Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-4078 [MISC] Remove the Celery execution transport from the workers, backend and SDK (#2284)

ali6154891fdcc5
diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.py
index 500c6d908..f0f3d16fb 100644
--- a/backend/pg_queue/models.py
+++ b/backend/pg_queue/models.py
@@ -257,12 +257,11 @@ class PgBatchDedup(models.Model):
class PgBarrierState(models.Model):
- """Per-execution fan-in barrier state for ``PgBarrier`` (the Postgres
- ``WORKER_BARRIER_BACKEND``).
+ """Per-execution fan-in barrier state for ``PgBarrier``.
- One row per in-flight barrier (keyed by ``execution_id``). The worker-side
- ``barrier_pg_decr_and_check`` link task atomically decrements ``remaining``
- and appends to ``results`` in a single ``UPDATE … RETURNING``; the task that
+ One row per in-flight barrier (keyed by ``execution_id``). Each header task
+ atomically decrements ``remaining`` in-body and appends to ``results`` in a
+ single ``UPDATE … RETURNING``; the task that
drives ``remaining`` to 0 dispatches the aggregating callback and deletes the
row. A header-task failure aborts the barrier by deleting the row outright
(``DELETE … RETURNING`` — atomic claim+teardown), so the callback can never
@@ -541,8 +540,7 @@ class PgOrchestrationClaim(models.Model):
``organization_id`` is stamped below — the reaper needs it for the org-scoped
status/mark API, exactly like ``pg_barrier_state``.
- Only written on the PG transport (``is_pg_transport``) — the Celery path never
- touches it. Managed=True / generated migration, extension-free — same posture
+ Managed=True / generated migration, extension-free — same posture
as the siblings.
"""
diff --git a/backend/pg_queue/tests/test_producer.py b/backend/pg_queue/tests/test_producer.py
index 246855a95..38ee9cc46 100644
--- a/backend/pg_queue/tests/test_producer.py
+++ b/backend/pg_queue/tests/test_producer.py
@@ -26,7 +26,7 @@ class TestEnqueueTask:
task_name="async_execute_bin",
queue="celery_api_deployments",
args=["org", "wf", "exec"],
- kwargs={"transport": "pg_queue"},
+ kwargs={"use_file_history": True},
org_id="org",
priority=5,
fairness={
@@ -44,7 +44,7 @@ class TestEnqueueTask:
assert msg["task_name"] == "async_execute_bin"
assert msg["queue"] == "celery_api_deployments"
assert msg["args"] == ["org", "wf", "exec"]
- assert msg["kwargs"] == {"transport": "pg_queue"}
+ assert msg["kwargs"] == {"use_file_history": True}
assert msg["fairness"]["workload_type"] == "api"
def test_uuid_args_kwargs_are_json_coerced(self):
diff --git a/backend/workflow_manager/internal_api_views.py b/backend/workflow_manager/internal_api_views.py
index c54199f16..15559a71c 100644
--- a/backend/workflow_manager/internal_api_views.py
+++ b/backend/workflow_manager/internal_api_views.py
@@ -18,7 +18,7 @@ from rest_framework.decorators import api_view
from rest_framework.response import Response
from tool_instance_v2.models import ToolInstance
-from unstract.core.data_models import WorkflowTransport
+from unstract.core.data_models import LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUE
from workflow_manager.workflow_v2.enums import ExecutionStatus
from workflow_manager.workflow_v2.models import Workflow, WorkflowExecution
@@ -326,9 +326,8 @@ def create_workflow_execution(request):
"execution_id": str(execution.id),
"status": execution.status,
"execution_log_id": execution.execution_log_id, # Return for workers to use
- # Wire field the workers still branch on. PG is the only
- # transport now; the field goes when those branches do.
- "transport": WorkflowTransport.PG_QUEUE.value,
+ # Rolling-deploy shim: see LEGACY_TRANSPORT_KEY.
+ LEGACY_TRANSPORT_KEY: LEGACY_TRANSPORT_VALUE,
}
)
diff --git a/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py b/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py
index 3ee117480..b51bb85e7 100644
--- a/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py
+++ b/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py
@@ -21,7 +21,7 @@ class TestDispatchOrchestratorTask:
task_id = WorkflowHelper._dispatch_orchestrator_task(
queue=CeleryQueue.CELERY_API_DEPLOYMENTS,
args=["a"],
- kwargs={"transport": "pg_queue"},
+ kwargs={"use_file_history": True},
org_schema="org1",
)
# bare msg_id, no "pg:" prefix — matches the worker PgDispatchHandle.id
diff --git a/backend/workflow_manager/workflow_v2/tests/test_legacy_transport_shim.py b/backend/workflow_manager/workflow_v2/tests/test_legacy_transport_shim.py
new file mode 100644
index 000000000..3d68a1dbb
--- /dev/null
+++ b/backend/workflow_manager/workflow_v2/tests/test_legacy_transport_shim.py
@@ -0,0 +1,85 @@
+"""Rolling-deploy shim for UN-4078.
+
+Nothing reads the ``transport`` key any more, but a pre-UN-4078 worker treats an
+absent key as "celery" and publishes to a RabbitMQ with no consumers. The backend
+must keep writing it for one release; delete this test with the writes.
+
+DB-free: the model, context, and ``time.sleep`` are mocked.
+"""
+
+from unittest.mock import MagicMock, patch
+
+import json
+
+from django.test import RequestFactory
+
+from workflow_manager.internal_api_views import create_workflow_execution
+from workflow_manager.workflow_v2.enums import ExecutionStatus
+from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper
+
+_MOD = "workflow_manager.workflow_v2.workflow_helper"
+_VIEWS = "workflow_manager.internal_api_views"
+
+
+class TestOrchestratorPayloadCarriesLegacyTransport:
+ def test_dispatch_kwargs_include_pg_queue_transport(self):
+ exec_row = MagicMock(status=ExecutionStatus.EXECUTING.value)
+
+ with (
+ patch(f"{_MOD}.UserContext") as user_ctx,
+ patch(f"{_MOD}.StateStore") as state_store,
+ patch(f"{_MOD}.time"),
+ patch(f"{_MOD}.WorkflowExecution") as wf_exec,
+ patch.object(
+ WorkflowHelper, "_dispatch_orchestrator_task", return_value="1"
+ ) as dispatch,
+ patch.object(WorkflowHelper, "_record_dispatch_handle"),
+ ):
+ user_ctx.get_organization_identifier.return_value = "org1"
+ state_store.get.return_value = None
+ wf_exec.objects.get.return_value = exec_row
+
+ WorkflowHelper.execute_workflow_async(
+ workflow_id="wf-1",
+ execution_id="exec-1",
+ hash_values_of_files={},
+ )
+
+ dispatch.assert_called_once()
+ assert dispatch.call_args.kwargs["kwargs"]["transport"] == "pg_queue"
+
+
+class TestCreateWorkflowExecutionResponseCarriesLegacyTransport:
+ """The scheduler reads this response body.
+
+ A pre-UN-4078 scheduler does ``workflow_execution.get("transport",
+ DEFAULT_WORKFLOW_TRANSPORT)`` against exactly this dict, so dropping the key
+ sends it down the Celery branch.
+ """
+
+ def test_response_includes_pg_queue_transport(self):
+ execution = MagicMock(
+ id="exec-1",
+ status=ExecutionStatus.PENDING.value,
+ execution_log_id="log-1",
+ )
+
+ with (
+ patch(f"{_VIEWS}.Organization") as organization,
+ patch(f"{_VIEWS}.Workflow") as workflow,
+ patch(f"{_VIEWS}.WorkflowExecution") as wf_exec,
+ ):
+ organization.objects.get.return_value = MagicMock()
+ workflow.objects.get.return_value = MagicMock()
+ wf_exec.objects.create.return_value = execution
+
+ request = RequestFactory().post(
+ "/internal/v1/workflow-execution/",
+ data=json.dumps({"workflow_id": "wf-1", "total_files": 1}),
+ content_type="application/json",
+ HTTP_X_ORGANIZATION_ID="org-1",
+ )
+ response = create_workflow_execution(request)
+
+ assert response.data["transport"] == "pg_queue"
+ assert response.data["execution_id"] == "exec-1"
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index da5ad380c..2c926d0dc 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -27,7 +27,11 @@ from utils.local_context import StateStore
from utils.user_context import UserContext
from backend.celery_service import app as celery_app
-from unstract.core.data_models import WorkflowTransport, WorkloadType
+from unstract.core.data_models import (
+ LEGACY_TRANSPORT_KEY,
+ LEGACY_TRANSPORT_VALUE,
+ WorkloadType,
+)
from unstract.workflow_execution.enums import LogStage
from workflow_manager.endpoint_v2.destination import DestinationConnector
from workflow_manager.endpoint_v2.dto import FileHash
@@ -71,7 +75,11 @@ EXECUTION_EXCLUDED_PARAMS = {
"hitl_queue_name",
"hitl_packet_id",
"custom_data",
- "transport",
+ # Still carried on the dispatch payload as a rolling-deploy shim
+ # (LEGACY_TRANSPORT_KEY), so it must never reach the legacy
+ # ``execute_workflow`` signature, where it would raise as an unexpected kwarg.
+ # Named through the constant so the shim's removal checklist finds this site.
+ LEGACY_TRANSPORT_KEY,
}
@@ -620,9 +628,8 @@ class WorkflowHelper:
"hitl_queue_name": hitl_queue_name,
"hitl_packet_id": hitl_packet_id,
"custom_data": custom_data,
- # Wire field the workers still branch on. PG is the only
- # transport now; the field goes when those branches do.
- "transport": WorkflowTransport.PG_QUEUE.value,
+ # Rolling-deploy shim: see LEGACY_TRANSPORT_KEY.
+ LEGACY_TRANSPORT_KEY: LEGACY_TRANSPORT_VALUE,
}
dispatch_handle = cls._dispatch_orchestrator_task(
queue=queue,
diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.py
index 00073ee2a..4ba675d54 100644
--- a/unstract/core/src/unstract/core/data_models.py
+++ b/unstract/core/src/unstract/core/data_models.py
@@ -200,92 +200,28 @@ class SourceConnectionType(str, Enum):
API = "API"
-class WorkflowTransport(str, Enum):
- """Transport a single workflow execution rides end-to-end.
-
- The migration unit is the *execution*, not the task: every stage of a
- coupled pipeline (async_execute → file-batch fan-out → callback) must run
- on one transport, decided once at execution creation and carried in the
- task payload (see ``9e-design.md``). ``CELERY`` is the legacy default;
- ``PG_QUEUE`` is the bespoke Postgres queue.
- """
-
- CELERY = "celery"
- PG_QUEUE = "pg_queue"
-
-
-# Default transport when none is carried on the payload.
+# Rolling-deploy shim (UN-4078). Nothing BRANCHES on this key any more, but it is
+# still written for one release, and one site still matches it by name (the
+# EXECUTION_EXCLUDED_PARAMS entry below). A pre-UN-4078 worker treats an absent
+# ``transport`` as "celery" and publishes the fan-out / callback to RabbitMQ, which
+# has no consumers, so during the rollout window an old pod receiving a new
+# producer's payload would strand the execution.
#
-# ⚠️ KNOWN HAZARD, DELIBERATELY NOT FIXED HERE — the safe direction has inverted.
-#
-# CELERY was the safe fallback while Celery had consumers: a payload that lost the
-# field degraded onto a working substrate. With the Celery fleet at zero that is no
-# longer true. A payload defaulting to "celery" now selects CeleryChordBarrier in
-# `_barrier_for_transport`, skips the PG orchestration-claim and terminal-redelivery
-# guards, and publishes the fan-out to a broker nothing drains — and because no PG
-# rows are written, neither the reaper nor the undispatched sweep can see the
-# stranded execution. Reachable on a rolling deploy, where a not-yet-upgraded
-# backend still emits `transport: "celery"`.
-#
-# Flipping this constant to PG_QUEUE is the right fix and it is a one-line change,
-# but it is NOT a one-line consequence: it breaks 19 tests across 6 files
-# (test_pg_barrier, test_chord_sites_characterisation, test_barrier,
-# test_dispatch_with_callback, test_dispatch_sites_characterisation,
-# test_workflow_context_transport), every one of which relies on the implicit
-# Celery default — most of them characterising PgBarrier's Celery-link branch,
-# which is deferred-deletion code. Rewriting those to state their transport
-# explicitly is correct, but it is a behaviour change that deserves its own review
-# rather than riding along in a documentation sweep.
-#
-# Until then the mitigation is that every producer sets the field explicitly
-# (workflow_helper.py and internal_api_views.py both hardcode PG_QUEUE), so the
-# default is only reachable via version skew.
-DEFAULT_WORKFLOW_TRANSPORT = WorkflowTransport.CELERY.value
-
-
-def normalize_transport(value: object, *, logger: Any = None, context: str = "") -> str:
- """Coerce an inbound transport value to a known ``WorkflowTransport`` value.
-
- The transport crosses untrusted boundaries — a task payload, PG JSONB, an
- older backend that omits the field — so a missing/garbage value must never
- route an execution onto an unknown substrate. This **fails closed**: an
- unrecognized value (``None``, ``"celary"``, ``""``) logs a warning (when a
- ``logger`` is given) and falls back to :data:`DEFAULT_WORKFLOW_TRANSPORT`
- — see the hazard note on that constant. A recognized value passes through as
- its canonical string.
-
- Coerce-not-raise is deliberate, and differs from how
- ``WorkflowContextData.workflow_type`` is validated: ``transport`` has an
- explicit safe default by design, so a bad value should degrade rather than
- crash the execution. What changed with UN-4046 is WHICH default is safe — see
- the note on the constant. A rolling deploy where an older backend still emits
- ``"celery"`` is the live case: that value is recognized, so it passes through
- here untouched and the guards keyed on it are skipped. This function only
- protects the missing/garbage case. ``context`` is an optional suffix for the
- log line (e.g. an ``exec:<id>`` tag) to make a version-skew warning traceable.
- """
- try:
- return WorkflowTransport(value).value
- except ValueError:
- if logger is not None:
- logger.warning(
- "Unrecognized workflow transport %r%s; falling back to %r",
- value,
- context,
- DEFAULT_WORKFLOW_TRANSPORT,
- )
- return DEFAULT_WORKFLOW_TRANSPORT
-
-
-def is_pg_transport(transport: str | None) -> bool:
- """True if ``transport`` is the Postgres-queue transport.
-
- Single source for "what counts as PG transport" — centralises the
- ``== WorkflowTransport.PG_QUEUE.value`` comparison scattered across the
- worker fan-out / barrier code, and the seam to extend if a second
- PG-family transport is ever added.
- """
- return transport == WorkflowTransport.PG_QUEUE.value
+# REMOVAL CHECKLIST for the release after UN-4078, once no pre-UN-4078 worker can
+# still be running (`grep -rn LEGACY_TRANSPORT_ ` finds every item but the last):
+# 1. the four writes — backend ``WorkflowHelper`` dispatch payload, backend
+# ``create_workflow_execution`` response, ``workers/scheduler/tasks.py``
+# dispatch kwargs, ``PgBarrier`` callback descriptor
+# 2. ``CallbackDescriptor.transport`` — workers/queue_backend/barrier.py
+# 3. the ``LEGACY_TRANSPORT_KEY`` entry in ``EXECUTION_EXCLUDED_PARAMS`` —
+# backend/workflow_manager/workflow_v2/workflow_helper.py
+# 4. these two constants
+# 5. the gating tests — workers/tests/test_legacy_transport_shim.py,
+# workers/tests/test_pg_barrier.py, the shim cases in
+# workers/tests/test_dispatch_sites_characterisation.py, and
+# backend/workflow_manager/workflow_v2/tests/test_legacy_transport_shim.py
+LEGACY_TRANSPORT_KEY = "transport"
+LEGACY_TRANSPORT_VALUE = "pg_queue"
class WorkloadType(StrEnum):
diff --git a/unstract/sdk1/src/unstract/sdk1/execution/__init__.py b/unstract/sdk1/src/unstract/sdk1/execution/__init__.py
index fa70c8882..a91602dcb 100644
--- a/unstract/sdk1/src/unstract/sdk1/execution/__init__.py
+++ b/unstract/sdk1/src/unstract/sdk1/execution/__init__.py
@@ -1,5 +1,4 @@
from unstract.sdk1.execution.context import ExecutionContext
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.sdk1.execution.executor import BaseExecutor
from unstract.sdk1.execution.orchestrator import ExecutionOrchestrator
from unstract.sdk1.execution.registry import ExecutorRegistry
@@ -8,7 +7,6 @@ from unstract.sdk1.execution.result import ExecutionResult
__all__ = [
"BaseExecutor",
"ExecutionContext",
- "ExecutionDispatcher",
"ExecutionOrchestrator",
"ExecutionResult",
"ExecutorRegistry",
diff --git a/unstract/sdk1/src/unstract/sdk1/execution/dispatcher.py b/unstract/sdk1/src/unstract/sdk1/execution/dispatcher.py
deleted file mode 100644
index 25a5da1f0..000000000
--- a/unstract/sdk1/src/unstract/sdk1/execution/dispatcher.py
+++ /dev/null
@@ -1,316 +0,0 @@
-"""Execution dispatcher for sending Celery tasks to executor workers.
-
-The dispatcher is the caller-side component used by both:
-- Structure tool Celery task (workflow path)
-- PromptStudioHelper (IDE path)
-
-It sends ``execute_extraction`` tasks to the ``executor`` queue.
-Three dispatch modes are available:
-
-- ``dispatch()``: Send and block until result (synchronous).
-- ``dispatch_async()``: Fire-and-forget, returns task_id for polling.
-- ``dispatch_with_callback()``: Fire-and-forget with Celery ``link``
- / ``link_error`` callbacks for post-processing.
-"""
-
-from __future__ import annotations
-
-import logging
-import os
-from typing import TYPE_CHECKING, Any
-
-if TYPE_CHECKING:
- from celery import Signature
- from celery.result import AsyncResult
- from unstract.sdk1.execution.context import ExecutionContext
-
-from unstract.sdk1.execution.result import ExecutionResult
-
-logger = logging.getLogger(__name__)
-
-# Constants matching workers/shared/enums values.
-# Defined here to avoid an SDK1 → workers package dependency.
-_TASK_NAME = "execute_extraction"
-
-# Queue-per-executor prefix. Each executor gets its own Celery queue
-# named ``celery_executor_{executor_name}``, derived automatically
-# from ``ExecutionContext.executor_name``.
-_QUEUE_PREFIX = "celery_executor_"
-
-# Caller-side timeout (seconds) for AsyncResult.get().
-# This controls how long the *caller* waits for the executor to
-# finish — distinct from the executor worker's
-# ``EXECUTOR_TASK_TIME_LIMIT`` which controls how long the
-# *worker* allows a task to run.
-#
-# Resolution order (matches workers convention):
-# 1. Explicit ``timeout`` parameter on dispatch()
-# 2. ``EXECUTOR_RESULT_TIMEOUT`` env var
-# 3. Hardcoded default (3600s)
-#
-# The default (3600s) is intentionally <= the executor worker's
-# ``task_time_limit`` default (also 3600s) so the caller never
-# waits longer than the worker allows the task to run.
-_DEFAULT_TIMEOUT_ENV = "EXECUTOR_RESULT_TIMEOUT"
-_DEFAULT_TIMEOUT = 3600 # 1 hour — matches executor worker default
-_NO_CELERY_APP_MSG = "No Celery app configured on ExecutionDispatcher"
-
-
-class ExecutionDispatcher:
- """Dispatches execution to executor worker via Celery task.
-
- Usage::
-
- dispatcher = ExecutionDispatcher(celery_app=app)
- result = dispatcher.dispatch(context, timeout=120)
-
- Fire-and-forget::
-
- task_id = dispatcher.dispatch_async(context)
-
- Fire-and-forget with callbacks::
-
- from celery import signature
-
- task = dispatcher.dispatch_with_callback(
- context,
- on_success=signature("my_success_task", args=[...], queue="q"),
- on_error=signature("my_error_task", args=[...], queue="q"),
- )
- """
-
- def __init__(self, celery_app: object | None = None) -> None:
- """Initialize the dispatcher.
-
- Args:
- celery_app: A Celery application instance. Required
- for dispatching tasks. Can be ``None`` only if
- set later via ``celery_app`` attribute.
- """
- self._app = celery_app
-
- @staticmethod
- def _get_queue(executor_name: str) -> str:
- """Derive the Celery queue name from *executor_name*.
-
- Convention: ``celery_executor_{executor_name}``.
- Adding a new executor automatically gets its own queue —
- no registry change needed.
- """
- return f"{_QUEUE_PREFIX}{executor_name}"
-
- @staticmethod
- def _build_send_kwargs(
- context: ExecutionContext,
- queue: str,
- *,
- headers: dict[str, Any] | None = None,
- link: Signature | None = None,
- link_error: Signature | None = None,
- task_id: str | None = None,
- ) -> dict[str, Any]:
- """Common ``send_task`` kwargs builder for the three dispatch modes.
-
- Optional kwargs are dropped when ``None`` so the resulting call
- shape matches the minimum-arity form Celery expects for ``link``
- / ``link_error`` handling and so callers that pass no headers
- see the same on-wire call as before headers existed.
- """
- out: dict[str, Any] = {
- "args": [context.to_dict()],
- "queue": queue,
- }
- # Treat falsy (None, ``{}``) as drop. An empty header dict is
- # never legitimate — ``FairnessKey.as_header()`` always returns
- # a populated mapping — so silently forwarding ``{}`` would
- # document a producer-side build bug rather than catch it.
- if headers:
- out["headers"] = headers
- if link is not None:
- out["link"] = link
- if link_error is not None:
- out["link_error"] = link_error
- if task_id is not None:
- out["task_id"] = task_id
- return out
-
- def dispatch(
- self,
- context: ExecutionContext,
- timeout: int | None = None,
- headers: dict[str, Any] | None = None,
- ) -> ExecutionResult:
- """Dispatch context as a Celery task and wait for result.
-
- Args:
- context: ExecutionContext to dispatch.
- timeout: Max seconds to wait. ``None`` reads from
- the ``EXECUTOR_RESULT_TIMEOUT`` env var,
- falling back to 3600s.
- headers: Optional Celery message headers (routing metadata
- out-of-band of the task body — see
- ``queue_backend.fairness.FAIRNESS_HEADER_NAME``
- (currently ``"x-fairness-key"``) for the canonical
- constant). Caller is responsible for the header schema.
- Falsy values (``None``, ``{}``) are dropped — the
- downstream ``send_task`` call is invoked without
- ``headers=`` so the on-wire shape matches the
- no-headers baseline.
-
- Returns:
- ExecutionResult from the executor.
-
- Raises:
- ValueError: If no Celery app is configured.
- """
- if self._app is None:
- raise ValueError(_NO_CELERY_APP_MSG)
-
- if timeout is None:
- timeout = int(os.environ.get(_DEFAULT_TIMEOUT_ENV, _DEFAULT_TIMEOUT))
-
- queue = self._get_queue(context.executor_name)
- logger.info(
- "Dispatching execution: executor=%s operation=%s"
- " run_id=%s request_id=%s timeout=%ss queue=%s",
- context.executor_name,
- context.operation,
- context.run_id,
- context.request_id,
- timeout,
- queue,
- )
-
- send_kwargs = self._build_send_kwargs(context, queue, headers=headers)
- async_result = self._app.send_task(_TASK_NAME, **send_kwargs)
- logger.info(
- "Task sent: celery_task_id=%s, waiting for result...",
- async_result.id,
- )
-
- try:
- # disable_sync_subtasks=False: safe because the executor task
- # runs on a separate worker pool (worker-v2) — no deadlock
- # risk even when dispatch() is called from inside a Django
- # Celery task.
- result_dict = async_result.get(
- timeout=timeout,
- disable_sync_subtasks=False,
- )
- except Exception as exc:
- logger.error(
- "Dispatch failed: executor=%s operation=%s run_id=%s error=%s",
- context.executor_name,
- context.operation,
- context.run_id,
- exc,
- )
- return ExecutionResult.failure(
- error=f"{type(exc).__name__}: {exc}",
- )
-
- return ExecutionResult.from_dict(result_dict)
-
- def dispatch_async(
- self,
- context: ExecutionContext,
- headers: dict[str, Any] | None = None,
- ) -> str:
- """Dispatch without waiting. Returns task_id for polling.
-
- Args:
- context: ExecutionContext to dispatch.
- headers: Optional Celery message headers (see ``dispatch``).
-
- Returns:
- The Celery task ID (use with ``AsyncResult`` to poll).
-
- Raises:
- ValueError: If no Celery app is configured.
- """
- if self._app is None:
- raise ValueError(_NO_CELERY_APP_MSG)
-
- queue = self._get_queue(context.executor_name)
- logger.info(
- "Dispatching async execution: executor=%s "
- "operation=%s run_id=%s request_id=%s queue=%s",
- context.executor_name,
- context.operation,
- context.run_id,
- context.request_id,
- queue,
- )
-
- send_kwargs = self._build_send_kwargs(context, queue, headers=headers)
- async_result = self._app.send_task(_TASK_NAME, **send_kwargs)
- return async_result.id
-
- def dispatch_with_callback(
- self,
- context: ExecutionContext,
- on_success: Signature | None = None,
- on_error: Signature | None = None,
- task_id: str | None = None,
- headers: dict[str, Any] | None = None,
- ) -> AsyncResult:
- """Fire-and-forget dispatch with Celery link callbacks.
-
- Sends the task to the executor queue and returns immediately.
- When the executor task completes, Celery invokes the
- ``on_success`` callback (via ``link``). If the executor task
- raises an exception, Celery invokes ``on_error`` (via
- ``link_error``).
-
- Args:
- context: ExecutionContext to dispatch.
- on_success: A Celery ``Signature`` invoked on success.
- Receives ``(result_dict,)`` as first positional arg
- followed by the signature's own args.
- on_error: A Celery ``Signature`` invoked on failure.
- Receives ``(failed_task_uuid,)`` as first positional
- arg followed by the signature's own args.
- task_id: Optional pre-generated Celery task ID. Useful
- when the caller needs to know the task ID before
- dispatch (e.g. to include it in callback kwargs).
- headers: Optional Celery message headers (see ``dispatch``).
-
- Returns:
- The ``AsyncResult`` from ``send_task``. Callers can
- use ``.id`` for task tracking but should NOT call
- ``.get()`` (that would block, defeating the purpose).
-
- Raises:
- ValueError: If no Celery app is configured.
- """
- if self._app is None:
- raise ValueError(_NO_CELERY_APP_MSG)
-
- queue = self._get_queue(context.executor_name)
- logger.info(
- "Dispatching with callback: executor=%s "
- "operation=%s run_id=%s request_id=%s "
- "on_success=%s on_error=%s queue=%s",
- context.executor_name,
- context.operation,
- context.run_id,
- context.request_id,
- on_success,
- on_error,
- queue,
- )
-
- send_kwargs = self._build_send_kwargs(
- context,
- queue,
- headers=headers,
- link=on_success,
- link_error=on_error,
- task_id=task_id,
- )
- async_result = self._app.send_task(_TASK_NAME, **send_kwargs)
- logger.info(
- "Task sent with callbacks: celery_task_id=%s",
- async_result.id,
- )
- return async_result
diff --git a/unstract/sdk1/tests/test_execution.py b/unstract/sdk1/tests/test_execution.py
index 374734597..1dcb8b18a 100644
--- a/unstract/sdk1/tests/test_execution.py
+++ b/unstract/sdk1/tests/test_execution.py
@@ -5,7 +5,6 @@ import logging
import os
import tempfile
from typing import Any, Self
-from unittest.mock import MagicMock
import pytest
from unstract.sdk1.constants import LogLevel, ToolEnv
@@ -15,7 +14,6 @@ from unstract.sdk1.execution.context import (
ExecutionSource,
Operation,
)
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.sdk1.execution.executor import BaseExecutor
from unstract.sdk1.execution.orchestrator import ExecutionOrchestrator
from unstract.sdk1.execution.registry import ExecutorRegistry
@@ -585,362 +583,6 @@ class TestExecutionOrchestrator:
assert result.error == "LLM rate limited"
-# ---- Phase 1F: ExecutionDispatcher ----
-
-
-class TestExecutionDispatcher:
- """Tests for ExecutionDispatcher (mocked Celery)."""
-
- def _make_context(self, **overrides: Any) -> ExecutionContext: # noqa: ANN401
- defaults: dict[str, Any] = {
- "executor_name": "legacy",
- "operation": "extract",
- "run_id": "run-1",
- "execution_source": "tool",
- "request_id": "req-1",
- }
- defaults.update(overrides)
- return ExecutionContext(**defaults)
-
- def _make_mock_app(
- self,
- result_dict: dict[str, Any] | None = None,
- side_effect: Exception | None = None,
- task_id: str = "celery-task-123",
- ) -> MagicMock:
- """Create a mock Celery app with send_task configured."""
- mock_app = MagicMock()
- mock_async_result = MagicMock()
- mock_async_result.id = task_id
-
- if side_effect is not None:
- mock_async_result.get.side_effect = side_effect
- else:
- mock_async_result.get.return_value = (
- result_dict
- if result_dict is not None
- else {"success": True, "data": {}, "metadata": {}}
- )
-
- mock_app.send_task.return_value = mock_async_result
- return mock_app
-
- def test_dispatch_sends_task_and_returns_result(
- self: Self,
- ) -> None:
- """dispatch() sends task to executor queue and returns result."""
- result_dict = {
- "success": True,
- "data": {"extracted_text": "hello"},
- "metadata": {},
- }
- mock_app = self._make_mock_app(result_dict=result_dict)
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- result = dispatcher.dispatch(ctx, timeout=60)
-
- assert result.success is True
- assert result.data == {"extracted_text": "hello"}
-
- # Verify send_task was called correctly
- mock_app.send_task.assert_called_once_with(
- "execute_extraction",
- args=[ctx.to_dict()],
- queue="celery_executor_legacy",
- )
- mock_app.send_task.return_value.get.assert_called_once_with(
- timeout=60, disable_sync_subtasks=False
- )
-
- def test_dispatch_uses_default_timeout(self: Self) -> None:
- """dispatch() without timeout uses default (3600s)."""
- mock_app = self._make_mock_app()
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- dispatcher.dispatch(ctx)
-
- mock_app.send_task.return_value.get.assert_called_once_with(
- timeout=3600, disable_sync_subtasks=False
- )
-
- def test_dispatch_timeout_from_env(
- self: Self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- """dispatch() reads timeout from EXECUTOR_RESULT_TIMEOUT env."""
- monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "120")
- mock_app = self._make_mock_app()
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- dispatcher.dispatch(ctx)
-
- mock_app.send_task.return_value.get.assert_called_once_with(
- timeout=120, disable_sync_subtasks=False
- )
-
- def test_dispatch_explicit_timeout_overrides_env(
- self: Self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- """Explicit timeout parameter overrides env var."""
- monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "120")
- mock_app = self._make_mock_app()
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- dispatcher.dispatch(ctx, timeout=30)
-
- mock_app.send_task.return_value.get.assert_called_once_with(
- timeout=30, disable_sync_subtasks=False
- )
-
- def test_dispatch_timeout_returns_failure(
- self: Self,
- ) -> None:
- """TimeoutError from AsyncResult.get() is wrapped in failure."""
- mock_app = self._make_mock_app(side_effect=TimeoutError("Task timed out"))
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- result = dispatcher.dispatch(ctx, timeout=1)
-
- assert result.success is False
- assert "TimeoutError" in result.error
-
- def test_dispatch_generic_exception_returns_failure(
- self: Self,
- ) -> None:
- """Any exception from AsyncResult.get() becomes a failure."""
- mock_app = self._make_mock_app(side_effect=RuntimeError("broker down"))
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- result = dispatcher.dispatch(ctx, timeout=10)
-
- assert result.success is False
- assert "RuntimeError" in result.error
- assert "broker down" in result.error
-
- def test_dispatch_async_returns_task_id(self: Self) -> None:
- """dispatch_async() returns the Celery task ID."""
- mock_app = self._make_mock_app(task_id="task-xyz-789")
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- task_id = dispatcher.dispatch_async(ctx)
-
- assert task_id == "task-xyz-789"
- mock_app.send_task.assert_called_once_with(
- "execute_extraction",
- args=[ctx.to_dict()],
- queue="celery_executor_legacy",
- )
-
- def test_dispatch_no_app_raises_value_error(
- self: Self,
- ) -> None:
- """dispatch() without celery_app raises ValueError."""
- dispatcher = ExecutionDispatcher(celery_app=None)
- ctx = self._make_context()
-
- with pytest.raises(ValueError, match="No Celery app"):
- dispatcher.dispatch(ctx)
-
- def test_dispatch_async_no_app_raises_value_error(
- self: Self,
- ) -> None:
- """dispatch_async() without celery_app raises ValueError."""
- dispatcher = ExecutionDispatcher(celery_app=None)
- ctx = self._make_context()
-
- with pytest.raises(ValueError, match="No Celery app"):
- dispatcher.dispatch_async(ctx)
-
- def test_dispatch_failure_result_from_executor(
- self: Self,
- ) -> None:
- """Executor failure is deserialized correctly."""
- result_dict = {
- "success": False,
- "data": {},
- "metadata": {},
- "error": "LLM adapter timeout",
- }
- mock_app = self._make_mock_app(result_dict=result_dict)
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- result = dispatcher.dispatch(ctx, timeout=60)
-
- assert result.success is False
- assert result.error == "LLM adapter timeout"
-
- def test_dispatch_context_serialized_correctly(
- self: Self,
- ) -> None:
- """The full ExecutionContext is serialized in the task args."""
- mock_app = self._make_mock_app()
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context(
- executor_name="agentic_table",
- operation="agentic_extraction",
- organization_id="org-42",
- executor_params={"schema": {"name": "str"}},
- )
-
- dispatcher.dispatch(ctx, timeout=60)
-
- sent_args = mock_app.send_task.call_args
- context_dict = sent_args[1]["args"][0]
-
- assert context_dict["executor_name"] == "agentic_table"
- assert context_dict["operation"] == "agentic_extraction"
- assert context_dict["organization_id"] == "org-42"
- assert context_dict["executor_params"] == {"schema": {"name": "str"}}
-
- # ---- Phase 5A: dispatch_with_callback ----
-
- def test_dispatch_with_callback_sends_link_and_link_error(
- self: Self,
- ) -> None:
- """dispatch_with_callback() passes on_success as link, on_error as link_error."""
- mock_app = self._make_mock_app(task_id="cb-task-001")
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- on_success = MagicMock(name="on_success_sig")
- on_error = MagicMock(name="on_error_sig")
-
- result = dispatcher.dispatch_with_callback(
- ctx, on_success=on_success, on_error=on_error
- )
-
- assert result.id == "cb-task-001"
- mock_app.send_task.assert_called_once_with(
- "execute_extraction",
- args=[ctx.to_dict()],
- queue="celery_executor_legacy",
- link=on_success,
- link_error=on_error,
- )
-
- def test_dispatch_with_callback_success_only(
- self: Self,
- ) -> None:
- """dispatch_with_callback() with only on_success omits link_error."""
- mock_app = self._make_mock_app(task_id="cb-task-002")
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- on_success = MagicMock(name="on_success_sig")
-
- dispatcher.dispatch_with_callback(ctx, on_success=on_success)
-
- call_kwargs = mock_app.send_task.call_args
- assert call_kwargs[1]["link"] is on_success
- assert "link_error" not in call_kwargs[1]
-
- def test_dispatch_with_callback_error_only(
- self: Self,
- ) -> None:
- """dispatch_with_callback() with only on_error omits link."""
- mock_app = self._make_mock_app(task_id="cb-task-003")
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- on_error = MagicMock(name="on_error_sig")
-
- dispatcher.dispatch_with_callback(ctx, on_error=on_error)
-
- call_kwargs = mock_app.send_task.call_args
- assert "link" not in call_kwargs[1]
- assert call_kwargs[1]["link_error"] is on_error
-
- def test_dispatch_with_callback_no_callbacks(
- self: Self,
- ) -> None:
- """dispatch_with_callback() with no callbacks sends plain task."""
- mock_app = self._make_mock_app(task_id="cb-task-004")
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- result = dispatcher.dispatch_with_callback(ctx)
-
- assert result.id == "cb-task-004"
- call_kwargs = mock_app.send_task.call_args
- assert "link" not in call_kwargs[1]
- assert "link_error" not in call_kwargs[1]
-
- def test_dispatch_with_callback_returns_async_result(
- self: Self,
- ) -> None:
- """dispatch_with_callback() returns the AsyncResult object (not just task_id)."""
- mock_app = self._make_mock_app(task_id="cb-task-005")
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- result = dispatcher.dispatch_with_callback(ctx)
-
- # Returns the full AsyncResult, not just the id string
- assert result is mock_app.send_task.return_value
- assert result.id == "cb-task-005"
-
- def test_dispatch_with_callback_no_app_raises_value_error(
- self: Self,
- ) -> None:
- """dispatch_with_callback() without celery_app raises ValueError."""
- dispatcher = ExecutionDispatcher(celery_app=None)
- ctx = self._make_context()
-
- with pytest.raises(ValueError, match="No Celery app"):
- dispatcher.dispatch_with_callback(ctx)
-
- def test_dispatch_with_callback_context_serialized(
- self: Self,
- ) -> None:
- """dispatch_with_callback() serializes context correctly."""
- mock_app = self._make_mock_app()
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context(
- operation="answer_prompt",
- executor_params={"prompt_key": "p1"},
- )
-
- dispatcher.dispatch_with_callback(ctx, on_success=MagicMock())
-
- sent_args = mock_app.send_task.call_args
- context_dict = sent_args[1]["args"][0]
- assert context_dict["operation"] == "answer_prompt"
- assert context_dict["executor_params"] == {"prompt_key": "p1"}
-
- def test_dispatch_with_callback_custom_task_id(
- self: Self,
- ) -> None:
- """dispatch_with_callback() passes custom task_id to send_task."""
- mock_app = self._make_mock_app(task_id="pre-gen-id-123")
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- dispatcher.dispatch_with_callback(ctx, task_id="pre-gen-id-123")
-
- call_kwargs = mock_app.send_task.call_args
- assert call_kwargs[1]["task_id"] == "pre-gen-id-123"
-
- def test_dispatch_with_callback_no_task_id_omits_kwarg(
- self: Self,
- ) -> None:
- """dispatch_with_callback() without task_id doesn't pass task_id."""
- mock_app = self._make_mock_app()
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
- ctx = self._make_context()
-
- dispatcher.dispatch_with_callback(ctx)
-
- call_kwargs = mock_app.send_task.call_args
- assert "task_id" not in call_kwargs[1]
-
-
# ---- Phase 1G: ExecutorToolShim ----
# Note: ExecutorToolShim lives in workers/executor/ but the tests
# import it directly via sys.path manipulation since the workers
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py
index 3fc3a2017..fcfad8836 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py
@@ -49,6 +49,18 @@ EXECUTE_TASK = "execute_extraction"
# Mirror the SDK's queue-per-executor convention so the PG executor queue name
# matches the Celery one (the worker-pg-executor consumer subscribes to these).
QUEUE_PREFIX = "celery_executor_"
+
+
+def queue_for_executor(executor_name: str) -> str:
+ """The queue an executor's dispatches land on.
+
+ One definition for the three dispatch sites below (and for the tests, which
+ import this rather than re-deriving the rule) so the wire contract
+ ``worker-pg-executor`` subscribes to cannot drift between them.
+ """
+ return f"{QUEUE_PREFIX}{executor_name}"
+
+
# Caller-side wait default — mirrors the SDK dispatcher (EXECUTOR_RESULT_TIMEOUT env,
# else 3600s) so a PG-routed caller waits exactly as long as a Celery one.
DEFAULT_TIMEOUT_ENV = "EXECUTOR_RESULT_TIMEOUT"
@@ -152,7 +164,7 @@ class PgExecutionDispatcher:
"""
timeout = _resolve_timeout(timeout)
reply_key = str(uuid.uuid4())
- queue = f"{QUEUE_PREFIX}{context.executor_name}"
+ queue = queue_for_executor(context.executor_name)
org = str(getattr(context, "organization_id", "") or "")
try:
self._transport.enqueue(
@@ -249,7 +261,7 @@ class PgExecutionDispatcher:
failure is observable even if the caller swallows it.
"""
task_id = str(uuid.uuid4())
- queue = f"{QUEUE_PREFIX}{context.executor_name}"
+ queue = queue_for_executor(context.executor_name)
org = str(getattr(context, "organization_id", "") or "")
try:
self._transport.enqueue(
@@ -294,7 +306,7 @@ class PgExecutionDispatcher:
prompt-studio views do, in their own try/except).
"""
task_id = task_id or str(uuid.uuid4())
- queue = f"{QUEUE_PREFIX}{context.executor_name}"
+ queue = queue_for_executor(context.executor_name)
org = str(getattr(context, "organization_id", "") or "")
success_spec = signature_to_continuation(on_success)
error_spec = signature_to_continuation(on_error)
diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py
index a07ef7b7b..4d4b56b3b 100644
--- a/workers/api-deployment/tasks.py
+++ b/workers/api-deployment/tasks.py
@@ -27,12 +27,9 @@ from shared.workflow.execution.tool_validation import validate_workflow_tool_ins
from worker import app
from unstract.core.data_models import (
- DEFAULT_WORKFLOW_TRANSPORT,
ExecutionStatus,
FileHashData,
WorkerFileData,
- is_pg_transport,
- normalize_transport,
)
from unstract.core.worker_models import ApiDeploymentResultStatus
@@ -422,15 +419,6 @@ def _run_workflow_api(
WorkflowHelper.process_input_files() methods.
"""
total_files = len(hash_values_of_files)
- # Resolve transport ONCE, up front from the authoritative kwargs source, so
- # the failure handler can rely on it even if orchestration fails before the
- # fan-out (the PR-1 seam; the fan-out honours it → PG path = PgBarrier).
- # Fail-closed to celery on any unrecognised value.
- transport = normalize_transport(
- kwargs.get("transport", DEFAULT_WORKFLOW_TRANSPORT),
- logger=logger,
- context=f" [exec:{execution_id}]",
- )
# TOOL VALIDATION: Validate tool instances before API workflow orchestration
# Get workflow execution context to retrieve tool instances
@@ -686,20 +674,15 @@ def _run_workflow_api(
file_processing_callback_queue = _get_callback_queue_name_api()
# Route through ``WorkflowOrchestrationUtils.create_chord_execution``
- # (which delegates to ``CeleryChordBarrier`` under the hood) instead of
- # calling ``chord(...)`` directly. Same Celery semantics, but the
- # ``Barrier`` abstraction lets a future Phase 6b swap to
- # ``RedisDecrBarrier`` behind a flag without touching this call site.
- # The header tasks + callback now carry the fairness slot for
- # downstream PG Queue routing — closes the chord-fairness gap noted
- # in Phase 5.1.
- # Hoist FairnessKey to a local so the chord path and the
+ # rather than reaching for a barrier directly, so both fan-out call
+ # sites share one dispatch point. The header tasks + callback carry the
+ # fairness slot for downstream routing.
+ # Hoist FairnessKey to a local so the fan-out path and the
# zero-batch fallback can't drift apart on the slot's shape.
api_fairness = FairnessKey(
org_id=str(schema_name),
workload_type=WorkloadType.API,
)
- # transport was resolved up front (see top of _run_workflow_api).
result = WorkflowOrchestrationUtils.create_chord_execution(
batch_tasks=batch_tasks,
callback_task_name="process_batch_callback_api",
@@ -711,7 +694,6 @@ def _run_workflow_api(
callback_queue=file_processing_callback_queue,
app_instance=app,
fairness=api_fairness,
- transport=transport,
)
if result is None:
@@ -810,36 +792,27 @@ def _run_workflow_api(
except Exception as e:
error_message = f"Error while processing files: {str(e)}"
- # On PG, surface the error to the UI + reconcile file counters so a failed
- # run reads "N failed" not "N in progress"; the wrapped update inside the
+ # Surface the error to the UI + reconcile file counters so a failed run
+ # reads "N failed" not "N in progress"; the wrapped update inside the
# helper guarantees a status-update failure can't mask `e` or skip the
- # re-raise below (see record_pg_orchestration_failure). The Celery branch
- # is the original bare update, untouched.
- if is_pg_transport(transport):
- error_logger = None
- try:
- error_logger = WorkerWorkflowLogger.create_for_api_workflow(
- execution_id=str(execution_id),
- organization_id=str(schema_name),
- pipeline_id=str(pipeline_id) if pipeline_id else None,
- )
- except Exception as log_error:
- logger.warning(f"Failed to build UI logger: {log_error}")
- WorkflowOrchestrationUtils.record_pg_orchestration_failure(
- api_client=api_client,
- execution_id=execution_id,
- total_files=total_files,
- error_message=error_message,
- logger=logger,
- workflow_logger=error_logger,
- )
- else:
- # Update execution to ERROR status matching Django pattern
- api_client.update_workflow_execution_status(
- execution_id=execution_id,
- status=ExecutionStatus.ERROR.value,
- error_message=error_message,
+ # re-raise below (see record_pg_orchestration_failure).
+ error_logger = None
+ try:
+ error_logger = WorkerWorkflowLogger.create_for_api_workflow(
+ execution_id=str(execution_id),
+ organization_id=str(schema_name),
+ pipeline_id=str(pipeline_id) if pipeline_id else None,
)
+ except Exception as log_error:
+ logger.warning(f"Failed to build UI logger: {log_error}")
+ WorkflowOrchestrationUtils.record_pg_orchestration_failure(
+ api_client=api_client,
+ execution_id=execution_id,
+ total_files=total_files,
+ error_message=error_message,
+ logger=logger,
+ workflow_logger=error_logger,
+ )
logger.error(f"Execution {execution_id} failed: {str(e)}", exc_info=True)
raise
diff --git a/workers/callback/tasks.py b/workers/callback/tasks.py
index 94297ccd4..8259fff59 100644
--- a/workers/callback/tasks.py
+++ b/workers/callback/tasks.py
@@ -365,7 +365,7 @@ def _update_execution_status_unified(
aggregated_results: dict[str, Any],
organization_id: str,
error_message: str | None = None,
- is_pg: bool = False,
+ raise_on_failure: bool = True,
) -> dict[str, Any]:
"""Unified workflow execution status update for all callback types.
@@ -379,6 +379,9 @@ def _update_execution_status_unified(
aggregated_results: Aggregated file processing results
organization_id: Organization context
error_message: Optional error message for failed executions
+ raise_on_failure: Re-raise a failed status write (the default) so PG's
+ at-least-once redelivery retries it. Pass ``False`` only where the
+ caller is already handling a failure and a raise would mask it.
Returns:
Execution update result dictionary
@@ -420,17 +423,16 @@ def _update_execution_status_unified(
logger.error(
f"Failed to update execution status for {execution_id}: {e}", exc_info=True
)
- # On the PG path this finalization write is the single terminalizing step.
- # Swallowing a failure here strands the execution in EXECUTING forever: the
- # consumer acks/deletes the message, the barrier row is already gone, and
- # the reaper has no handle — every safety net is silently defeated. Re-raise
- # so PG's at-least-once vt-redelivery retries the write and, if it keeps
- # failing, poison-drops the batch to a terminal ERROR. A post-success
- # redelivery is idempotency-guarded by _callback_already_ran (skips once
- # COMPLETED; ERROR/STOPPED are intentionally re-run, so a failed-write
- # ERROR execution simply retries the write). The Celery path keeps the
- # legacy swallow (its chord retry covers it) so this is a PG-only change.
- if is_pg:
+ # This finalization write is the single terminalizing step. Swallowing a
+ # failure here strands the execution in EXECUTING forever: the consumer
+ # acks/deletes the message, the barrier row is already gone, and the reaper
+ # has no handle — every safety net is silently defeated. Re-raise so PG's
+ # at-least-once vt-redelivery retries the write and, if it keeps failing,
+ # poison-drops the batch to a terminal ERROR. A post-success redelivery is
+ # idempotency-guarded by _callback_already_ran (skips once COMPLETED;
+ # ERROR/STOPPED are intentionally re-run, so a failed-write ERROR execution
+ # simply retries the write).
+ if raise_on_failure:
raise
# Return error result instead of re-raising to maintain callback flow
return {
@@ -1430,9 +1432,10 @@ def _process_batch_callback_core(
# Initialize performance optimizations
_initialize_performance_managers()
- # PG at-least-once duplicate guard (see _callback_already_ran).
- # Popped BEFORE parameter extraction so the marker never flows into the context.
- is_pg = bool(kwargs.pop(PG_TRANSPORT_CALLBACK_KWARG, False))
+ # Wire-compat only: PgBarrier stamps this marker on every callback it fires.
+ # Nothing branches on it since UN-4078 (PG is the only transport), but it is
+ # popped BEFORE parameter extraction so it never flows into the context.
+ kwargs.pop(PG_TRANSPORT_CALLBACK_KWARG, None)
# Extract and validate all parameters using single source of truth
context = _extract_callback_parameters(task_instance, results, kwargs)
@@ -1460,7 +1463,7 @@ def _process_batch_callback_core(
# completed — don't re-fire webhooks / re-count billing. Reuses the
# status from _extract_callback_parameters' fetch (no extra call).
# (Returns through the outer finally, which closes the api_client.)
- if is_pg and _callback_already_ran(context.execution_status):
+ if _callback_already_ran(context.execution_status):
logger.warning(
f"PG callback: execution {context.execution_id} already COMPLETED "
f"— a previous callback finalized it; skipping duplicate side "
@@ -1486,7 +1489,6 @@ def _process_batch_callback_core(
aggregated_results=aggregated_results,
organization_id=context.organization_id,
error_message=None,
- is_pg=is_pg,
)
# Handle pipeline updates using unified function (non-API deployment)
pipeline_result = _handle_pipeline_updates_unified(
@@ -1566,6 +1568,9 @@ def _process_batch_callback_core(
{"error": str(e)[:500]},
context.organization_id,
error_message=str(e)[:500],
+ # Already handling a failure: a raise here would replace
+ # the original error with the status-write error.
+ raise_on_failure=False,
)
logger.info(
f"Marked execution {context.execution_id} as failed using unified function"
@@ -1643,8 +1648,9 @@ def process_batch_callback_api(
execution_id = kwargs.get("execution_id")
pipeline_id = kwargs.get("pipeline_id")
organization_id = kwargs.get("organization_id")
- # PG at-least-once duplicate guard marker (see _callback_already_ran).
- is_pg = bool(kwargs.pop(PG_TRANSPORT_CALLBACK_KWARG, False))
+ # Wire-compat only: stamped by PgBarrier on every callback it fires. Nothing
+ # branches on it since UN-4078; popped so it never flows into the context.
+ kwargs.pop(PG_TRANSPORT_CALLBACK_KWARG, None)
if not execution_id:
raise ValueError("execution_id is required in kwargs")
@@ -1677,7 +1683,7 @@ def process_batch_callback_api(
# prior delivery already completed must skip its side effects. Reuses the
# status just fetched — no extra round-trip. (Returns through the finally
# below, which closes the api_client.)
- if is_pg and _callback_already_ran(workflow_execution.get("status")):
+ if _callback_already_ran(workflow_execution.get("status")):
logger.warning(
f"PG API callback: execution {execution_id} already COMPLETED — a "
f"previous callback finalized it; skipping duplicate side effects "
@@ -1746,7 +1752,6 @@ def process_batch_callback_api(
final_status=execution_status,
aggregated_results=aggregated_results,
organization_id=organization_id,
- is_pg=is_pg,
)
# Create minimal context for unified pipeline handling
diff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.py
index 4b2f02bfc..95d745211 100644
--- a/workers/file_processing/structure_tool_task.py
+++ b/workers/file_processing/structure_tool_task.py
@@ -1,17 +1,16 @@
-"""Structure tool Celery task — Phase 3 of executor migration.
+"""Structure tool task — Phase 3 of executor migration.
-Replaces the Docker-container-based StructureTool.run() with a Celery
-task that runs in the file_processing worker. Instead of PromptTool
-HTTP calls to prompt-service, it uses ExecutionDispatcher to send
-operations to the executor worker via Celery.
+Replaces the Docker-container-based StructureTool.run() with a task that runs in
+the file_processing worker. Instead of PromptTool HTTP calls to prompt-service,
+it dispatches operations to the executor worker through the executor RPC.
Before (Docker-based):
File Processing Worker → WorkflowExecutionService → ToolSandbox
→ Docker container → StructureTool.run() → PromptTool (HTTP) → prompt-service
-After (Celery-based):
+After:
File Processing Worker → WorkerWorkflowExecutionService
- → execute_structure_tool task → ExecutionDispatcher
+ → execute_structure_tool task → executor RPC (PG queue)
→ executor worker → LegacyExecutor
"""
@@ -276,10 +275,10 @@ def _merge_agentic_metrics(
@worker_task(bind=True, name=str(TaskName.EXECUTE_STRUCTURE_TOOL))
def execute_structure_tool(self, params: dict) -> dict:
- """Execute structure tool as a Celery task.
+ """Execute structure tool as a task.
Replicates StructureTool.run() from tools/structure/src/main.py
- but uses ExecutionDispatcher instead of PromptTool HTTP calls.
+ but uses the executor RPC instead of PromptTool HTTP calls.
Args:
params: Dict with keys described in the Phase 3 plan.
diff --git a/workers/file_processing/tasks.py b/workers/file_processing/tasks.py
index 05b71ffaa..764953834 100644
--- a/workers/file_processing/tasks.py
+++ b/workers/file_processing/tasks.py
@@ -54,7 +54,6 @@ from unstract.core.data_models import (
FileHashData,
PreCreatedFileData,
WorkerFileData,
- WorkflowTransport,
)
from unstract.core.worker_models import (
ApiDeploymentResultStatus,
@@ -239,10 +238,9 @@ class _TerminalExecutionSkip(Exception):
resurrect the execution back to EXECUTING and re-run its files (double LLM /
destination write). The batch is skipped instead.
- Gated to the PG path (``is_pg``): a no-op on Celery, so the existing Celery
- flow is behaviorally unchanged. It's PG-specific because the **reaper** that
- marks an in-flight execution terminal (and tears its barrier down) is a
- PG-only mechanism — the Celery chord has no equivalent.
+ The resurrection this prevents exists because the **reaper** can mark an
+ in-flight execution terminal (and tear its barrier down) while a batch for it
+ is still in flight or waiting to be redelivered.
This is a validate-first *narrowing*, not a transactional guarantee: the
status read and the later ``EXECUTING`` write are not atomic, so a reaper
@@ -260,20 +258,16 @@ class _TerminalExecutionSkip(Exception):
def _raise_if_execution_terminal(
- workflow_execution: dict[str, Any], execution_id: str, *, is_pg: bool
+ workflow_execution: dict[str, Any], execution_id: str
) -> None:
- """PG-path-only validate-first guard: refuse to (re)process a batch whose
- execution is already terminal. See :class:`_TerminalExecutionSkip`.
-
- No-op on the Celery path (``is_pg=False``) — the resurrection this prevents
- is PG-specific, so gating leaves the Celery flow behaviorally unchanged. A
- missing or unrecognized status is treated as non-terminal (fail open —
- proceed as normal), but both are logged on the PG path so a degraded
- execution-fetch response or a new/renamed status isn't silently masked (an
- unrecognized terminal-ish state would otherwise let a stale batch through).
+ """Validate-first guard: refuse to (re)process a batch whose execution is
+ already terminal. See :class:`_TerminalExecutionSkip`.
+
+ A missing or unrecognized status is treated as non-terminal (fail open —
+ proceed as normal), but both are logged so a degraded execution-fetch
+ response or a new/renamed status isn't silently masked (an unrecognized
+ terminal-ish state would otherwise let a stale batch through).
"""
- if not is_pg:
- return
status = workflow_execution.get("status")
if not status:
# Fail open (proceed), but surface it — a missing status is an
@@ -319,17 +313,13 @@ def _terminal_skip_result(batch_data: FileBatchData) -> dict[str, Any]:
def _run_batch_stages(
- file_batch_data: dict[str, Any], celery_task_id: str, *, is_pg: bool
+ file_batch_data: dict[str, Any], celery_task_id: str
) -> dict[str, Any]:
"""The actual batch work (validate → setup → pre-create → process → compile).
- Transport-agnostic: identical on the Celery chord path and the PG
- fire-and-forget path. The task instance isn't needed here — its only use
- (deriving ``celery_task_id``) happens in the caller. Returns the
- JSON-serialisable batch result.
-
- ``is_pg`` (keyword-only, no default so a PG call site can't silently forget
- it) forwards to the terminal guard; see :class:`_TerminalExecutionSkip`.
+ The task instance isn't needed here — its only use (deriving
+ ``celery_task_id``) happens in the caller. Returns the JSON-serialisable
+ batch result.
"""
# Step 1: Validate and parse input data
batch_data = _validate_and_parse_batch_data(file_batch_data)
@@ -338,7 +328,7 @@ def _run_batch_stages(
# stale/redelivered batch for an already-terminal execution — see
# :class:`_TerminalExecutionSkip`.
try:
- context = _setup_execution_context(batch_data, celery_task_id, is_pg=is_pg)
+ context = _setup_execution_context(batch_data, celery_task_id)
except _TerminalExecutionSkip as skip:
logger.warning(
f"[exec:{skip.execution_id}] Skipping batch of "
@@ -374,11 +364,11 @@ def _process_file_batch_core(
Args:
task_instance: The Celery task instance (self)
file_batch_data: Dictionary that will be converted to FileBatchData dataclass
- barrier_context: Present only on the 9e PG fire-and-forget path — carries
- ``execution_id`` / ``batch_index`` / ``callback_descriptor`` so the
- batch claims its slot and runs the barrier decrement in-body (a
- PG-consumed task fires no Celery ``.link``). ``None`` on the Celery
- chord path, where the chord ``.link`` drives the decrement instead.
+ barrier_context: Carries ``execution_id`` / ``batch_index`` /
+ ``callback_descriptor`` so the batch claims its slot and runs the
+ barrier decrement in-body. Every fan-out injects it
+ (``PgBarrier._dispatch_header_pg``), so ``None`` means a malformed
+ payload — see the guard below, which logs and still runs the batch.
Returns:
Dictionary with successful_files and failed_files counts
@@ -388,17 +378,27 @@ def _process_file_batch_core(
)
if barrier_context is None:
- # Celery chord path — the chord's .link runs the decrement after this.
- # is_pg=False disables the terminal guard → Celery flow unchanged.
- return _run_batch_stages(file_batch_data, celery_task_id, is_pg=False)
-
- # PG fire-and-forget path — claim the batch (idempotent on redelivery), run
- # the stages, then decrement the barrier in-body / self-chain the callback.
- # is_pg=True enables the terminal guard (skip a stale/redelivered batch for a
- # reaper-recovered execution).
+ # Every fan-out injects ``_barrier_context`` (PgBarrier._dispatch_header_pg),
+ # so this means a malformed or hand-crafted payload. Run the batch — the
+ # work is still well-defined — but say so loudly: with no barrier to
+ # decrement, nothing will fire the aggregating callback and the execution
+ # will sit until the reaper reclaims it.
+ file_data = file_batch_data.get("file_data") or {}
+ logger.error(
+ f"[exec:{file_data.get('execution_id')}] "
+ f"[workflow:{file_data.get('workflow_id')}] process_file_batch received "
+ f"no _barrier_context for a batch of "
+ f"{len(file_batch_data.get('files') or [])} file(s); the batch will run "
+ "but no barrier will be decremented and the aggregating callback will "
+ "never fire. This indicates a malformed payload."
+ )
+ return _run_batch_stages(file_batch_data, celery_task_id)
+
+ # Claim the batch (idempotent on redelivery), run the stages, then decrement
+ # the barrier in-body / self-chain the callback.
return run_batch_with_barrier(
barrier_context,
- lambda: _run_batch_stages(file_batch_data, celery_task_id, is_pg=True),
+ lambda: _run_batch_stages(file_batch_data, celery_task_id),
)
@@ -425,10 +425,9 @@ def process_file_batch(
Args:
file_batch_data: Dictionary that will be converted to FileBatchData dataclass
- _barrier_context: Injected only when this task is dispatched onto the PG
- queue (9e fire-and-forget path) by ``PgBarrier`` — carries the barrier
- coordination context (``execution_id`` / ``batch_index`` /
- ``callback_descriptor``). Absent on the Celery chord path.
+ _barrier_context: Injected by ``PgBarrier`` on every fan-out — carries the
+ barrier coordination context (``execution_id`` / ``batch_index`` /
+ ``callback_descriptor``). Absent only on a malformed payload.
Returns:
Dictionary with successful_files and failed_files counts
@@ -470,24 +469,21 @@ def _validate_and_parse_batch_data(file_batch_data: dict[str, Any]) -> FileBatch
def _setup_execution_context(
- batch_data: FileBatchData, celery_task_id: str, *, is_pg: bool
+ batch_data: FileBatchData, celery_task_id: str
) -> WorkflowContextData:
"""Setup execution context with validation and API client initialization.
Args:
batch_data: Validated batch data
- celery_task_id: Celery task ID for tracking
- is_pg: Whether this batch runs on the PG (at-least-once) transport.
- Keyword-only, no default so a PG call site can't silently forget it.
- Enables the terminal guard; a no-op on Celery.
+ celery_task_id: Task ID for tracking
Returns:
WorkflowContextData containing type-safe execution context
Raises:
ValueError: If required context fields are missing
- _TerminalExecutionSkip: On the PG path, if the execution is already
- terminal (a stale/redelivered batch after reaper-recovery).
+ _TerminalExecutionSkip: If the execution is already terminal (a
+ stale/redelivered batch after reaper-recovery).
"""
# Extract context using dataclass
file_data = batch_data.file_data
@@ -527,10 +523,10 @@ def _setup_execution_context(
execution_context = execution_response.data
workflow_execution = execution_context.get("execution", {})
- # Validate-first terminal guard (PG only) — skip a stale/redelivered batch
+ # Validate-first terminal guard — skip a stale/redelivered batch
# for an already-terminal execution, before we touch status or process
# anything. See :class:`_TerminalExecutionSkip`.
- _raise_if_execution_terminal(workflow_execution, execution_id, is_pg=is_pg)
+ _raise_if_execution_terminal(workflow_execution, execution_id)
# Set LOG_EVENTS_ID in StateStore for WebSocket messaging (critical for UI logs)
# This enables the WorkerWorkflowLogger to send logs to the UI via WebSocket
@@ -599,18 +595,12 @@ def _setup_execution_context(
f"File history from fallback access for workflow {workflow_id}: use_file_history = {use_file_history}"
)
- # Create type-safe workflow context. Set the transport authoritatively from
- # is_pg (the batch-level fact) — WorkerFileData carries none — so the
- # destination layer can apply PG-only guards.
context_data = WorkflowContextData(
workflow_id=workflow_id,
workflow_name=workflow_name,
workflow_type=workflow_type,
execution_id=execution_id,
organization_context=org_context,
- transport=(
- WorkflowTransport.PG_QUEUE.value if is_pg else WorkflowTransport.CELERY.value
- ),
files={
f"file_{i}": file for i, file in enumerate(files)
}, # Convert list to dict format
@@ -875,7 +865,6 @@ def _process_individual_files(context: WorkflowContextData) -> WorkflowContextDa
workflow_file_execution_id=workflow_file_execution_id, # Pass pre-created ID
workflow_file_execution_object=workflow_file_execution_object, # Pass pre-created object
workflow_logger=workflow_logger, # Pass workflow logger for UI logging
- transport=context.transport, # Drives the PG-only destination guard
)
# Handle file processing result
@@ -1645,7 +1634,6 @@ def _process_file(
workflow_file_execution_id: str = None,
workflow_file_execution_object: Any = None,
workflow_logger: Any = None,
- transport: str | None = None,
) -> dict[str, Any]:
"""Process a single file matching Django backend _process_file pattern.
@@ -1659,8 +1647,6 @@ def _process_file(
file_hash: FileHashData instance with type-safe access
api_client: Internal API client
workflow_execution: Workflow execution context
- transport: Execution transport (celery | pg_queue); drives PG-only
- destination guards downstream.
Returns:
File execution result
@@ -1676,7 +1662,6 @@ def _process_file(
workflow_file_execution_id=workflow_file_execution_id,
workflow_file_execution_object=workflow_file_execution_object,
workflow_logger=workflow_logger,
- transport=transport,
)
@@ -2242,56 +2227,36 @@ def process_file_batch_resilient(
def process_file_batch_django_compat(
self, file_batch_data: dict[str, Any]
) -> dict[str, Any]:
- """Backward compatibility wrapper for Django backend task name.
+ """Backward compatibility wrapper for the old Django backend task name.
- This allows new workers to handle tasks sent from the old Django backend
- during the transition period when both systems are running.
+ Registered so a task sent under the pre-workers name is still recognised.
+ Its only producer was the in-backend Celery chord, which UN-4078 removed
+ along with the rest of the Celery transport.
+
+ It carries no ``_barrier_context``, so the batch would run with no claim
+ (no ``pg_batch_dedup`` marker, so a redelivery re-runs the whole batch: the
+ LLM spend twice, and a duplicate destination write when
+ ``use_file_history=False``) and with nothing to fire the aggregating
+ callback. Refuse instead — the same choice ``step_execution`` makes on the
+ backend — so the failure is visible rather than a silent double-spend.
Args:
- file_batch_data: File batch data from Django backend
+ file_batch_data: File batch data from the old Django backend
- Returns:
- Same result as process_file_batch
+ Raises:
+ RuntimeError: always.
"""
- logger.info(
- "Processing file batch via Django compatibility task name: "
- "workflow_manager.workflow_v2.file_execution_tasks.process_file_batch"
+ file_data = file_batch_data.get("file_data") or {}
+ raise RuntimeError(
+ f"[exec:{file_data.get('execution_id')}] "
+ f"[workflow:{file_data.get('workflow_id')}] the Django-compat task name "
+ "workflow_manager.workflow_v2.file_execution_tasks.process_file_batch is no "
+ "longer executable: it carries no barrier context, so the batch would run "
+ "unclaimed and no aggregating callback would fire. Its only producer was the "
+ "Celery chord removed in UN-4078. Dispatch process_file_batch through "
+ "PgBarrier instead."
)
- # Django compatibility: Calculate and apply manual review requirements
- # This replicates the MRQ logic that was originally in Django backend
- try:
- # Extract organization_id from Django backend data structure
- # Django sends: {files: [...], file_data: {organization_id: "...", ...}}
- file_data = file_batch_data.get("file_data", {})
- organization_id = file_data.get("organization_id")
-
- if not organization_id:
- logger.warning(
- "Django compatibility: No organization_id found in file_data, skipping MRQ calculation"
- )
- else:
- # Create organization-scoped API client
- api_client = create_api_client(organization_id)
-
- # Calculate manual review requirements
- mrq_flags = _calculate_manual_review_requirements(file_batch_data, api_client)
-
- # Enhance batch data with MRQ flags
- _enhance_batch_with_mrq_flags(file_batch_data, mrq_flags)
-
- logger.info(
- f"Django compatibility: Applied manual review flags to file batch for org {organization_id}"
- )
-
- except Exception as e:
- logger.warning(f"Django compatibility: Failed to calculate MRQ flags: {e}")
- raise
- # Continue processing without MRQ flags rather than failing
-
- # Delegate to the core implementation (same as main task)
- return _process_file_batch_core(self, file_batch_data)
-
# Helper functions for refactored _handle_file_processing_result
diff --git a/workers/general/tasks.py b/workers/general/tasks.py
index 7200a6589..861ddc7dd 100644
--- a/workers/general/tasks.py
+++ b/workers/general/tasks.py
@@ -56,13 +56,10 @@ from worker import app, config
# Import shared data models for type safety
from unstract.core.data_models import (
- DEFAULT_WORKFLOW_TRANSPORT,
ExecutionStatus,
FileBatchData,
FileHashData,
WorkerFileData,
- is_pg_transport,
- normalize_transport,
)
# Import common workflow utilities
@@ -143,16 +140,15 @@ def _log_batch_creation_statistics(
def _should_skip_duplicate_orchestration(
- execution_id: str, organization_id: str, transport: str, retries: int = 0
+ execution_id: str, organization_id: str, retries: int = 0
) -> bool:
- """PG-only: claim the execution's orchestration slot; ``True`` iff THIS
- delivery lost the claim (a duplicate / redelivered orchestration) and must
- no-op.
+ """Claim the execution's orchestration slot; ``True`` iff THIS delivery lost
+ the claim (a duplicate / redelivered orchestration) and must no-op.
``organization_id`` (the org schema_name) is stamped onto the claim row so the
reaper can call the org-scoped API when it recovers or GCs an orphan claim.
- On the PG queue the orchestration task can be redelivered — it runs longer
+ The orchestration task can be redelivered — it runs longer
than the consumer's visibility timeout and a sibling replica re-claims the
message — and re-running it re-arms the barrier (resets ``remaining`` and
wipes the dedup markers mid-flight) and dispatches a second set of batch
@@ -165,14 +161,7 @@ def _should_skip_duplicate_orchestration(
a prior attempt claimed and did not release it (a hard crash after the claim,
before the failure-path release) — a suppressed retry, logged at ERROR with an
errorId so it's distinguishable from a normal duplicate.
-
- No-op on Celery (returns ``False`` — always proceeds): Celery has no
- at-least-once redelivery, and the router owns terminal status there, so there
- is no double-orchestration to guard. Gated exactly like the terminal guard so
- the Celery path is behaviorally unchanged.
"""
- if not is_pg_transport(transport):
- return False
if try_claim_orchestration(execution_id, organization_id):
return False # first delivery — this replica owns the orchestration
if retries > 0:
@@ -213,7 +202,6 @@ def async_execute_bin_general(
pipeline_id: str | None = None,
log_events_id: str | None = None,
use_file_history: bool = False,
- transport: str = DEFAULT_WORKFLOW_TRANSPORT,
**kwargs: dict[str, Any],
) -> dict[str, Any]:
"""Lightweight general workflow execution task.
@@ -248,27 +236,27 @@ def async_execute_bin_general(
f"Starting general workflow execution for workflow {workflow_id}, execution {execution_id}"
)
- # Tracks whether THIS delivery won the PG orchestration claim, so the
- # error path releases the claim only when we actually acquired it (not, for
+ # Tracks whether THIS delivery won the orchestration claim, so the error
+ # path releases the claim only when we actually acquired it (not, for
# instance, when the claim's own INSERT raised). Set below only after the
- # gate returns "proceed" on the PG path.
+ # gate returns "proceed".
claimed_orchestration = False
try:
- # PG-only idempotency gate (claim-before-work): a duplicate /
- # redelivered orchestration no-ops here BEFORE any setup, barrier arm,
- # or batch dispatch — so exactly one delivery orchestrates the
- # execution even with replicas > 1. No-op on Celery.
+ # Idempotency gate (claim-before-work): a duplicate / redelivered
+ # orchestration no-ops here BEFORE any setup, barrier arm, or batch
+ # dispatch — so exactly one delivery orchestrates the execution even
+ # with replicas > 1.
if _should_skip_duplicate_orchestration(
- execution_id, schema_name, transport, self.request.retries
+ execution_id, schema_name, self.request.retries
):
return {
"status": "skipped_duplicate_orchestration",
"execution_id": execution_id,
"workflow_id": workflow_id,
}
- # Gate returned "proceed": on PG that means this delivery just won the
- # claim (Celery never claims), so the error path owns releasing it.
- claimed_orchestration = is_pg_transport(transport)
+ # Gate returned "proceed": this delivery just won the claim, so the
+ # error path owns releasing it.
+ claimed_orchestration = True
# Initialize execution context with shared utility
config, api_client = WorkerExecutionContext.setup_execution_context(
@@ -312,16 +300,11 @@ def async_execute_bin_general(
# to test) a redelivery could reach here, win a fresh claim, and re-arm +
# re-dispatch a finished execution — duplicate (costly) destination
# writes when use_file_history=False. Skip if already terminal, keeping
- # the freshly-won claim so it re-establishes the tombstone. PG only:
- # Celery has no redelivery, so the orchestrator only ever runs on a
- # non-terminal execution — the check is a no-op there.
- if (
- is_pg_transport(transport)
- and current_status in ExecutionStatus.terminal_values()
- ):
+ # the freshly-won claim so it re-establishes the tombstone.
+ if current_status in ExecutionStatus.terminal_values():
logger.warning(
f"Execution {execution_id} already terminal ({current_status}) "
- f"at orchestration entry — skipping re-orchestration (a PG "
+ f"at orchestration entry — skipping re-orchestration (a "
f"redelivery of a finished execution)."
)
return {
@@ -369,7 +352,6 @@ def async_execute_bin_general(
use_file_history,
scheduled,
schema_name,
- transport=transport,
**kwargs,
)
@@ -605,7 +587,6 @@ def _execute_general_workflow(
use_file_history: bool,
scheduled: bool,
schema_name: str,
- transport: str = DEFAULT_WORKFLOW_TRANSPORT,
**kwargs: dict[str, Any],
) -> dict[str, Any]:
"""Execute general workflow specific logic for ETL/TASK workflows.
@@ -628,12 +609,6 @@ def _execute_general_workflow(
"""
start_time = time.time()
- # Fail-closed coercion, for parity with the api/scheduler workers (which run
- # normalize_transport at their entry): a typo'd transport degrades to Celery
- # with a warning rather than silently routing onto an unknown substrate. The
- # coerced value feeds both WorkflowContextData and the fan-out below.
- transport = normalize_transport(transport, logger=logger)
-
logger.info("Executing general workflow logic for ETL/TASK workflow")
try:
@@ -672,7 +647,6 @@ def _execute_general_workflow(
"execution_mode": execution_mode,
},
is_scheduled=scheduled,
- transport=transport,
)
logger.info(
@@ -856,7 +830,6 @@ def _execute_general_workflow(
execution_mode=execution_mode,
use_file_history=use_file_history,
organization_id=api_client.organization_id,
- transport=transport,
**kwargs,
)
@@ -868,28 +841,17 @@ def _execute_general_workflow(
except Exception as e:
logger.error(f"Workflow execution failed: {e}")
- # On PG, surface the error to the UI + reconcile file counters so a
- # failed run reads "N failed" not "N in progress" (see
- # WorkflowOrchestrationUtils.record_pg_orchestration_failure). The
- # Celery branch below is the original status update, untouched.
- if is_pg_transport(transport):
- WorkflowOrchestrationUtils.record_pg_orchestration_failure(
- api_client=api_client,
- execution_id=execution_id,
- total_files=total_files,
- error_message=str(e),
- logger=logger,
- workflow_logger=workflow_logger,
- )
- else:
- try:
- api_client.update_workflow_execution_status(
- execution_id=execution_id,
- status=ExecutionStatus.ERROR.value,
- error_message=str(e),
- )
- except Exception as status_error:
- logger.warning(f"Failed to update error status: {status_error}")
+ # Surface the error to the UI + reconcile file counters so a failed
+ # run reads "N failed" not "N in progress" (see
+ # WorkflowOrchestrationUtils.record_pg_orchestration_failure).
+ WorkflowOrchestrationUtils.record_pg_orchestration_failure(
+ api_client=api_client,
+ execution_id=execution_id,
+ total_files=total_files,
+ error_message=str(e),
+ logger=logger,
+ workflow_logger=workflow_logger,
+ )
orchestration_result = WorkerTaskResponse.error_response(
execution_id=execution_id,
@@ -925,7 +887,6 @@ def _orchestrate_file_processing_general(
execution_mode: tuple | None,
use_file_history: bool,
organization_id: str,
- transport: str = DEFAULT_WORKFLOW_TRANSPORT,
**kwargs: dict[str, Any],
) -> dict[str, Any]:
"""Orchestrate file processing for general workflows using the same pattern as API worker.
@@ -1119,7 +1080,6 @@ def _orchestrate_file_processing_general(
org_id=organization_id,
workload_type=WorkloadType.NON_API,
),
- transport=transport,
)
if not result:
diff --git a/workers/queue_backend/__init__.py b/workers/queue_backend/__init__.py
index e148708a6..febd02104 100644
--- a/workers/queue_backend/__init__.py
+++ b/workers/queue_backend/__init__.py
@@ -1,128 +1,37 @@
"""Queue-backend seam.
-Single place where the substrate choice (Celery today; PG Queue later)
-lives. Both entry points are transparent passthroughs to Celery today.
-
-The ``Barrier`` substrate is runtime-selectable via the
-``WORKER_BARRIER_BACKEND`` env var:
-
-- ``chord`` (default) — ``CeleryChordBarrier``, the thin wrapper around
- ``celery.chord(header)(body)``. Byte-identical wire to the
- pre-abstraction code path; current production behaviour.
-- ``redis`` — ``RedisDecrBarrier``, the labs-design distributed-counter
- pattern (``DECR remaining:{exec_id}`` + per-task ``RPUSH``-based
- result aggregation). Replaces only the chord aggregation primitive;
- dispatch / retries / monitoring still ride on Celery.
-
-**Default-safety posture.** The env defaults to ``chord``, so a merge
-into ``main`` is zero-impact: a downstream env (``unstract-cloud``,
-self-hosted, etc.) that does not set ``WORKER_BARRIER_BACKEND`` runs
-the same substrate it ran yesterday. The new ``RedisDecrBarrier`` code
-is dormant until an operator explicitly flips the flag.
-
-Unknown values raise — operators get a loud error at module import
-time rather than silently falling back, which would mask a typo'd
-flag in a production env.
-
-**Queue-transport routing (separate axis).** ``dispatch()`` consults
-:func:`~queue_backend.routing.select_backend`, which returns PG
-unconditionally since UN-4046. This is orthogonal to the barrier choice
-above: the barrier is *how a batch's fan-in fires the callback*; the
-transport is *how messages travel*. It is emphatically NOT inert — every
-dispatch rides PG and needs its consumer running, or the message is
-durably enqueued and never executed, with no error at the producer.
-(It used to read a ``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list that was
-set nowhere, so everything defaulted to Celery; both are gone.)
+Single place where the queue substrate lives. Postgres is the only transport and
+:class:`~queue_backend.pg_barrier.PgBarrier` the only fan-in barrier.
+
+**History.** Both were once selectable. ``dispatch()`` read a
+``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list (set nowhere, so everything
+defaulted to Celery — removed in UN-4046), and the barrier substrate was chosen
+at import time by ``WORKER_BARRIER_BACKEND`` from ``chord`` /
+``redis`` / ``pg``. The Celery and Redis barriers, the env var and the
+``get_barrier()`` factory went with the Celery transport in UN-4078.
+
+**Every dispatch rides PG and needs its consumer running**, or the message is
+durably enqueued and never executed, with no error at the producer. The
+deployment chart (``validate-pg-worker-fleet.yaml``, in the closed-source
+``unstract-cloud`` repo — it is not in this tree) refuses a partial fleet at
+render time for exactly this reason.
"""
-import os
-from enum import StrEnum
-
-from .barrier import Barrier, BarrierHandle, CeleryChordBarrier
+from .barrier import Barrier, BarrierHandle
from .decorator import worker_task
from .dispatch import dispatch
from .fairness import FairnessKey
-from .pg_barrier import (
- PgBarrier,
- barrier_pg_abort,
- barrier_pg_decr_and_check,
-)
-from .redis_barrier import (
- RedisDecrBarrier,
- barrier_abort,
- barrier_decr_and_check,
-)
+from .pg_barrier import PgBarrier, barrier_pg_abort
from .routing import QueueBackend, select_backend
__all__ = [
"Barrier",
- "BarrierBackend",
"BarrierHandle",
- "CeleryChordBarrier",
"FairnessKey",
"PgBarrier",
"QueueBackend",
- "RedisDecrBarrier",
- "barrier_abort",
- "barrier_decr_and_check",
"barrier_pg_abort",
- "barrier_pg_decr_and_check",
"dispatch",
- "get_barrier",
"select_backend",
"worker_task",
]
-
-
-class BarrierBackend(StrEnum):
- """Enumeration of ``Barrier`` substrate choices.
-
- Used as the canonical reference for the ``WORKER_BARRIER_BACKEND``
- env value, the factory's dispatch table, and tests that flip the
- env. Subclassing ``StrEnum`` (Python 3.11+) means ``BarrierBackend.CHORD
- == "chord"`` at runtime, so the enum members can stand in anywhere
- a string is expected — including ``os.environ`` reads.
- """
-
- CHORD = "chord"
- REDIS = "redis"
- PG = "pg"
-
-
-def get_barrier() -> Barrier:
- """Return the ``Barrier`` implementation selected by env.
-
- Reads ``WORKER_BARRIER_BACKEND`` at call time so test harnesses
- that ``monkeypatch.setenv`` can flip the substrate per-test
- without a module reload. Production call sites construct the
- barrier once per process (module-level singleton in
- ``orchestration_utils.py``) so the env read happens at worker
- startup — flag flips require pod restart, same posture as every
- other ``WORKER_*`` env in the codebase.
-
- Default ``BarrierBackend.CHORD`` mirrors the current production
- substrate. An unknown value raises ``ValueError`` so a typo'd
- ``redis`` → ``rediz`` doesn't silently fall back to chord (which
- would invalidate the canary).
- """
- raw = os.getenv("WORKER_BARRIER_BACKEND", BarrierBackend.CHORD.value)
- try:
- backend = BarrierBackend(raw.strip().lower())
- except ValueError:
- valid = tuple(b.value for b in BarrierBackend)
- raise ValueError(
- f"WORKER_BARRIER_BACKEND={raw!r} is not a recognised "
- f"barrier backend. Valid values: {valid}. Unset the env var "
- f"to default to {BarrierBackend.CHORD.value!r}."
- ) from None
-
- if backend is BarrierBackend.CHORD:
- return CeleryChordBarrier()
- if backend is BarrierBackend.REDIS:
- return RedisDecrBarrier()
- if backend is BarrierBackend.PG:
- return PgBarrier()
- # Unreachable — StrEnum constructor would have raised above for
- # anything not in the enum. Defensive raise so the type checker
- # sees an exhaustive match.
- raise AssertionError(f"Unhandled BarrierBackend: {backend!r}")
diff --git a/workers/queue_backend/barrier.py b/workers/queue_backend/barrier.py
index 4695765de..0bc773c22 100644
--- a/workers/queue_backend/barrier.py
+++ b/workers/queue_backend/barrier.py
@@ -1,46 +1,30 @@
-"""Distributed barrier — abstraction over the chord-completion pattern.
-
-The two production chord call sites (``shared/workflow/execution/
-orchestration_utils.py`` and ``api-deployment/tasks.py``) need to wait
-for a fan-out of header tasks to all complete, then fire a callback
-with the aggregated results. Today that's implemented by Celery's
-``chord(header)(body)`` primitive, which is the highest-risk Celery
-construct at our scale — silent task drops at ~130K-task scale are
-documented in the PG Queue decision journey.
-
-This module provides the ``Barrier`` protocol and a ``CeleryChordBarrier``
-implementation that wraps the existing ``chord(...)`` call 1:1. Lifting
-both call sites to ``Barrier`` gives Phase 6b a place to land an
-alternative implementation (e.g. ``RedisDecrBarrier`` using the labs
-``DECR remaining:{exec_id}`` pattern, or — later in Phase 8 — a
-``PgBarrier`` using SKIP LOCKED) without touching the call sites a
-second time.
-
-**Phase 6a (this PR): wrapper only — behaviour-preserving uplift.**
-``CeleryChordBarrier.enqueue(...)`` produces byte-identical wire output
-to the direct ``chord(...)`` calls it replaces *when ``fairness=None``*.
-When a ``FairnessKey`` is passed (which both production call sites
-now do), each header task and the callback additionally carry an
-``x-fairness-key`` AMQP header. That's an additive wire-level change —
-consumers ignore unknown headers — same posture as Phase 5.1's
-``dispatch()`` plumbing. Mixed-version rolling deploys are safe in
-both directions: old workers ignore the new header; new workers
-handle messages without the header identically to today.
-
-Phase 6b (separate PR) will add ``RedisDecrBarrier`` as a second
-implementation, gated by ``WORKER_BARRIER_BACKEND`` env flag (default
-``chord``). Phase 6c is the rollout.
+"""Distributed barrier — the fan-out-then-callback seam.
+
+The two production fan-out call sites (``shared/workflow/execution/
+orchestration_utils.py`` and ``api-deployment/tasks.py``) need to wait for a
+fan-out of header tasks to all complete, then fire a callback with the
+aggregated results.
+
+This module holds the ``Barrier`` protocol and the types that cross the
+producer → queue → consumer boundary. The single implementation is
+:class:`~queue_backend.pg_barrier.PgBarrier`.
+
+**History.** This seam existed to let the substrate be swapped without touching
+the call sites twice. It carried three implementations: ``CeleryChordBarrier``
+(``celery.chord``, the original), ``RedisDecrBarrier`` (a ``DECR`` counter), and
+``PgBarrier``. Selection was runtime, via ``WORKER_BARRIER_BACKEND``. The first
+two, the env var, and the ``get_barrier()`` factory were deleted in UN-4078 once
+PG became the only transport. Nothing annotates against the protocol any more —
+both fan-out sites construct :class:`PgBarrier` concretely — so it survives only
+as documentation of the seam's shape, alongside the shared TypedDicts below,
+which ARE load-bearing. Deleting it is a separate post-decommission cleanup.
"""
from __future__ import annotations
import logging
import os
-from typing import TYPE_CHECKING, Any, NotRequired, Protocol, TypedDict
-
-from celery import chord
-
-from unstract.core.data_models import DEFAULT_WORKFLOW_TRANSPORT
+from typing import TYPE_CHECKING, Any, Literal, Protocol, TypedDict
from .fairness import FairnessKey
from .handle import BarrierHandle
@@ -82,9 +66,9 @@ def _positive_int_env(var: str, default: int, human_default: str) -> int:
def barrier_ttl_seconds() -> int:
"""Barrier TTL from ``WORKER_BARRIER_KEY_TTL_SECONDS`` (default 6h).
- Read at call time so tests can flip it. Invalid / non-positive values raise,
- matching ``get_barrier()``'s loud-on-misconfig posture — a TTL shorter than
- execution wall-clock would tear barriers down early (spurious behaviour).
+ Read at call time so tests can flip it. Invalid / non-positive values raise
+ rather than falling back — a TTL shorter than execution wall-clock would tear
+ barriers down early (spurious behaviour), so a garbled value must fail loud.
"""
return _positive_int_env(
"WORKER_BARRIER_KEY_TTL_SECONDS",
@@ -126,23 +110,24 @@ def barrier_stuck_timeout_seconds() -> int:
class CallbackDescriptor(TypedDict):
"""Serialisable aggregating-callback spec baked into a barrier link signature.
- Crosses a Celery serialisation boundary (producer → broker → worker), so the
- four-key contract is typed to catch a typo/rename before it surfaces as a
- remote ``KeyError`` mid-aggregation. Shared by both the Redis and PG
- backends. ``fairness_headers`` is ``None`` when the producer passed no key.
+ Crosses a serialisation boundary (producer → queue → worker), so the contract
+ is typed to catch a typo/rename before it surfaces as a remote ``KeyError``
+ mid-aggregation. ``fairness_headers`` is ``None`` when the producer passed no
+ key.
"""
task_name: str
kwargs: dict[str, Any]
queue: str
fairness_headers: dict[str, Any] | None
- # 9e: the WorkflowTransport the aggregating callback is fired on when the
- # barrier completes. Absent / ``None`` → legacy Celery dispatch
- # (``current_app.apply_async`` — the ``.link`` path). ``"pg_queue"`` → the
- # fire-and-forget PG path self-chains the callback via dispatch onto PG.
- # Named ``transport`` (a WorkflowTransport value, e.g. ``"pg_queue"``) — NOT
- # ``backend``, to avoid confusion with ``QueueBackend`` (``"pg"``).
- transport: NotRequired[str | None]
+ # Rolling-deploy shim, see LEGACY_TRANSPORT_KEY in unstract.core.data_models.
+ # REQUIRED for the shim release, not optional: a pre-UN-4078 consumer reads
+ # ``callback_descriptor.get("transport")`` and, finding nothing, fires the
+ # aggregating callback through Celery onto a broker with no consumers —
+ # stranding the execution. ``Literal`` rejects "celery", the one value that
+ # must never be written here, and makes the follow-up removal a type error at
+ # the write site rather than a grep.
+ transport: Literal["pg_queue"]
class BarrierContext(TypedDict):
@@ -184,8 +169,8 @@ class Barrier(Protocol):
Semantically: "enqueue these header tasks in parallel; when all
complete, fire this callback with the list of header results."
- The concrete substrate (Celery chord today, Redis DECR or PG
- SKIP LOCKED later) is hidden behind this protocol.
+ :class:`~queue_backend.pg_barrier.PgBarrier` (PG ``SKIP LOCKED``) is the sole
+ implementation since UN-4078.
"""
def enqueue(
@@ -197,15 +182,9 @@ class Barrier(Protocol):
callback_queue: str,
app_instance: Any,
fairness: FairnessKey | None = None,
- transport: str = DEFAULT_WORKFLOW_TRANSPORT,
) -> BarrierHandle | None:
"""Enqueue ``header_tasks`` and a single callback to fire on completion.
- ``transport`` is the per-execution transport (9e). Only ``PgBarrier``
- acts on it (``pg_queue`` → fire-and-forget PG fan-out instead of Celery
- ``.link``); the Celery/Redis substrates accept it for Protocol parity
- and ignore it (they are only ever reached on the ``celery`` transport).
-
``None`` is the **sole signal** that no work was enqueued — it
is returned exclusively when ``header_tasks`` is empty. Any
substrate-level failure (broker outage, serialisation error,
@@ -221,117 +200,9 @@ class Barrier(Protocol):
callback — same wire shape as ``dispatch()`` uses for bare
sites. Pass ``None`` for non-workflow-execution callers.
- ``app_instance`` is a Celery app today (the Protocol leaks
- this concrete dependency for the same reason the
- ``CeleryChordBarrier`` does — Phase 6b's RedisDecrBarrier will
- narrow this to a smaller Protocol). The argument is the
- ``app`` from the producer's task context and must expose
- ``.signature(name, kwargs=..., queue=..., headers=...)``.
+ ``app_instance`` is accepted for backward compatibility with the two
+ call sites and is unused by :class:`PgBarrier` — the header tasks are
+ dispatched onto the PG queue by name, not through a Celery app. It goes
+ with the protocol in the post-decommission cleanup.
"""
...
-
-
-class CeleryChordBarrier:
- """Thin wrapper around ``celery.chord``.
-
- Behaviour-preserving: the ``chord(header_tasks)(callback_signature)``
- call shape is the same as the inline ``chord(...)`` it replaces.
- Wire output is byte-identical to the pre-Barrier path when
- ``fairness=None``; when a ``FairnessKey`` is passed the only
- addition is the ``x-fairness-key`` AMQP header on each header
- task and the callback (additive wire-level change — consumers
- that don't read the header are unaffected, same backward-compat
- posture as Phase 5.1's ``dispatch()`` fairness plumbing).
-
- Future ``RedisDecrBarrier`` and ``PgBarrier`` implementations will
- live alongside this class and be selected by the
- ``WORKER_BARRIER_BACKEND`` env flag (Phase 6b).
- """
-
- def enqueue(
- self,
- header_tasks: list[Signature],
- *,
- callback_task_name: str,
- callback_kwargs: dict[str, Any],
- callback_queue: str,
- app_instance: Any,
- fairness: FairnessKey | None = None,
- transport: str = DEFAULT_WORKFLOW_TRANSPORT,
- ) -> BarrierHandle | None:
- """See :class:`Barrier.enqueue`."""
- del transport # Celery chord is only reached on the celery transport.
- # Empty-header guard goes FIRST so a zero-task run skips
- # signature construction / fairness serialisation entirely.
- # Any failure in those paths is now constrained to the
- # non-empty branch where the work is actually needed.
- if not header_tasks:
- execution_id = callback_kwargs.get("execution_id")
- pipeline_id = callback_kwargs.get("pipeline_id")
- logger.info(
- f"[exec:{execution_id}] [pipeline:{pipeline_id}] "
- "Zero header tasks detected — skipping barrier enqueue "
- "(parent should handle pipeline status updates directly)"
- )
- return None
-
- try:
- # Use ``fairness.as_header()`` — the single wire-shape
- # encoder, also used by ``dispatch.py``. Constructing the
- # header dict inline here would risk silent divergence
- # from the bare-dispatch path.
- fairness_headers = fairness.as_header() if fairness else None
-
- # Build the callback-signature kwargs explicitly. When
- # ``fairness=None`` the resulting call shape is
- # byte-identical to the pre-Barrier ``app.signature(name,
- # kwargs=..., queue=...)`` (no spurious ``headers=None``).
- signature_kwargs: dict[str, Any] = {
- "kwargs": callback_kwargs,
- "queue": callback_queue,
- }
- if fairness_headers:
- signature_kwargs["headers"] = fairness_headers
- callback_signature = app_instance.signature(
- callback_task_name, **signature_kwargs
- )
-
- # Stamp fairness onto each header signature via
- # ``Signature.clone().set(headers=...)`` rather than
- # in-place ``.set(...)`` on the caller's list. Cloning
- # avoids cross-tenant header leakage if a future retry
- # path or signature cache ever re-uses the original
- # ``header_tasks`` list with a different ``FairnessKey``.
- if fairness_headers:
- signed_header_tasks = [
- task.clone().set(headers=fairness_headers) for task in header_tasks
- ]
- else:
- signed_header_tasks = header_tasks
-
- result = chord(signed_header_tasks)(callback_signature)
-
- logger.info(
- f"Barrier enqueued via CeleryChordBarrier — "
- f"header_tasks={len(signed_header_tasks)}, "
- f"callback={callback_task_name}, "
- f"queue={callback_queue}"
- )
-
- return result
-
- except Exception:
- # ``logger.exception`` auto-attaches the traceback — needed
- # for debugging broker outages / serialisation failures
- # at the chord entry point. Add the readily-available
- # context so a Sentry/log triage gets execution/pipeline/
- # callback/queue identifiers alongside the stack.
- execution_id = callback_kwargs.get("execution_id")
- pipeline_id = callback_kwargs.get("pipeline_id")
- logger.exception(
- f"[exec:{execution_id}] [pipeline:{pipeline_id}] "
- f"Failed to enqueue barrier "
- f"(callback={callback_task_name}, queue={callback_queue}, "
- f"header_tasks={len(header_tasks)})"
- )
- raise
diff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.py
index d841ae0bc..03c822a8b 100644
--- a/workers/queue_backend/pg_barrier.py
+++ b/workers/queue_backend/pg_barrier.py
@@ -1,44 +1,41 @@
-"""PG fan-in barrier — Postgres substrate for the ``Barrier`` Protocol.
-
-Third ``WORKER_BARRIER_BACKEND`` option (``pg``) alongside ``chord`` and
-``redis``. Mirrors :class:`~queue_backend.redis_barrier.RedisDecrBarrier`
-exactly — same ``enqueue`` signature, same ``BarrierHandle | None`` contract,
-same fairness plumbing, same Celery-dispatched header tasks with
-``.link``/``.link_error`` — but moves the **aggregation** ("wait for N tasks,
-then fire the callback with their results") from a Redis ``DECR`` counter to a
-Postgres row. Selected at runtime by ``queue_backend.get_barrier``; default
-stays ``chord``.
-
-**Why a Postgres substrate.** It lets an execution coordinate in the *same*
-Postgres that holds the PG queue — no Redis (or RabbitMQ chord backend) needed
-for the fan-in. The transport for the header tasks themselves is unchanged
-(still Celery); only the coordination moves.
+"""PG fan-in barrier — the ``Barrier`` implementation.
+
+Coordinates "wait for N header tasks, then fire the callback with their
+results" on a Postgres row (``pg_barrier_state``) rather than a Celery chord or
+a Redis ``DECR`` counter. Both of those substrates were deleted with the Celery
+transport (UN-4078); this is the only barrier.
+
+**Why a Postgres substrate.** An execution coordinates in the *same* Postgres
+that holds the PG queue, so the fan-in needs no second datastore — and, unlike a
+broker-side or TTL'd-counter barrier, its state is a directly queryable row the
+reaper can recover.
**Wire model.**
1. ``enqueue``: UPSERT one ``pg_barrier_state`` row (``remaining = N``,
``results = []``, ``expires_at = now() + ttl``, ``last_progress_at = now()``)
- — the UPSERT clears any stale
- state from a prior run reusing the same ``execution_id``. Each header task is
- dispatched with
- ``.link(barrier_pg_decr_and_check)`` (success) and
- ``.link_error(barrier_pg_abort)`` (failure).
-2. Per-task success: ``barrier_pg_decr_and_check`` runs ONE atomic statement —
+ — the UPSERT clears any stale state from a prior run reusing the same
+ ``execution_id``. Each header task is then dispatched onto the PG queue with a
+ ``_barrier_context`` kwarg carrying ``execution_id`` / ``batch_index`` /
+ ``callback_descriptor``.
+2. Per-task success: a PG-consumed task fires no Celery ``.link``, so it claims
+ its batch and runs the decrement **in-body** (:func:`run_batch_with_barrier` →
+ :func:`_barrier_pg_decrement`) — ONE atomic statement,
``UPDATE … SET remaining = remaining - 1, results = results ||
jsonb_build_array(result) … RETURNING remaining, results``. The row lock
serialises concurrent decrements, so exactly one task observes ``remaining =
- 0``; that task dispatches the callback with the aggregated results, then
- deletes the row. (No Lua — a single ``UPDATE … RETURNING`` is atomic in
+ 0``; that task self-chains the callback onto PG with the aggregated results,
+ then deletes the row. (No Lua — a single ``UPDATE … RETURNING`` is atomic in
Postgres. The guarantee relies on each decrement committing in its own
transaction — do NOT batch decrements into a shared transaction, or the row
lock would hold and the serialisation that makes exactly one see 0 breaks.)
-3. Per-task failure: ``barrier_pg_abort`` runs as a ``link_error``. It tears the
- barrier down with ONE atomic statement (``DELETE … RETURNING`` in a single
+3. Per-task failure: :func:`barrier_pg_abort`, called in-body, tears the barrier
+ down with ONE atomic statement (``DELETE … RETURNING`` in a single
transaction): the row's existence is the dedup token, so N concurrent
failures collapse to a single cleanup, and a crash mid-abort rolls back
(leaving the row for a sibling to retry) — there is no claimed-but-not-deleted
- window. Mirrors chord's default error semantic (callback not invoked on
- header failure).
+ window. Preserves chord's old error semantic (callback not invoked on header
+ failure).
4. Stuck bound: ``last_progress_at`` is re-stamped to ``now()`` on enqueue AND on
every decrement, tracking when a batch last completed. The
:mod:`~queue_backend.pg_queue.reaper` marks the stranded execution ERROR once
@@ -57,9 +54,9 @@ succeeded. A failed task runs the abort (which deletes the row) instead of a
decrement, so the count never reaches 0 and a late in-flight decrement finds no
row → abandons. A decrement that drives ``remaining`` negative (expiry/replay)
also cleans up without firing. The callback is dispatched BEFORE the row is
-deleted, so a callback ``apply_async`` failure leaves the row (and its expiry)
-in place rather than stranding the execution; the post-dispatch delete is
-best-effort (logged, not raised) since the callback has already fired.
+deleted, so a callback dispatch failure leaves the row (and its expiry) in place
+rather than stranding the execution; the post-dispatch delete is best-effort
+(logged, not raised) since the callback has already fired.
"""
from __future__ import annotations
@@ -77,11 +74,7 @@ import psycopg2
import psycopg2.errors
import psycopg2.extensions
-from unstract.core.data_models import (
- DEFAULT_WORKFLOW_TRANSPORT,
- WorkflowTransport,
- is_pg_transport,
-)
+from unstract.core.data_models import LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUE
from .barrier import (
BarrierContext,
@@ -550,11 +543,35 @@ class _PgBarrierHandle:
id: str
+def build_callback_descriptor(
+ *,
+ task_name: str,
+ kwargs: dict[str, Any],
+ queue: str,
+ fairness_headers: dict[str, Any] | None,
+) -> CallbackDescriptor:
+ """Build the descriptor carried to the consumer that fires the callback.
+
+ A function rather than an inline literal so the rolling-deploy shim it stamps
+ (:data:`~unstract.core.data_models.LEGACY_TRANSPORT_KEY`) is pinned by a test
+ that needs no Postgres — the enqueue path's own tests are skipped wherever a
+ database is unavailable, which would let the shim be dropped against a green
+ required build. See ``tests/test_legacy_transport_shim.py``.
+ """
+ return {
+ "task_name": task_name,
+ "kwargs": kwargs,
+ "queue": queue,
+ "fairness_headers": fairness_headers,
+ # Rolling-deploy shim: see LEGACY_TRANSPORT_KEY.
+ LEGACY_TRANSPORT_KEY: LEGACY_TRANSPORT_VALUE,
+ }
+
+
class PgBarrier:
"""``Barrier`` implementation via a Postgres ``pg_barrier_state`` row.
- Drop-in for ``CeleryChordBarrier`` / ``RedisDecrBarrier`` from the call
- sites' perspective. See the module docstring for the wire model and the
+ The only implementation. See the module docstring for the wire model and the
failure-masking guards.
"""
@@ -567,7 +584,6 @@ class PgBarrier:
callback_queue: str,
app_instance: Any,
fairness: FairnessKey | None = None,
- transport: str = DEFAULT_WORKFLOW_TRANSPORT,
) -> BarrierHandle | None:
"""See :class:`queue_backend.barrier.Barrier.enqueue`.
@@ -575,16 +591,11 @@ class PgBarrier:
any substrate failure raises. ``app_instance`` is accepted for Protocol
parity but unused.
- ``transport`` selects the fan-out mode:
-
- - ``celery`` (default, the ``WORKER_BARRIER_BACKEND=pg`` legacy path):
- header tasks are Celery-dispatched with ``.link(barrier_pg_decr_and_check)``
- / ``.link_error(barrier_pg_abort)``; the link drives the decrement.
- - ``pg_queue`` (the fire-and-forget path): header tasks are dispatched onto
- the PG queue (``dispatch(backend=PG)``) with a ``_barrier_context`` kwarg
- carrying ``execution_id`` / ``batch_index`` / ``callback_descriptor`` —
- a PG-consumed task fires no ``.link``, so it claims its batch and runs
- the decrement in-body, self-chaining the callback at ``remaining`` → 0.
+ Header tasks are dispatched onto the PG queue (``dispatch(backend=PG)``)
+ with a ``_barrier_context`` kwarg carrying ``execution_id`` /
+ ``batch_index`` / ``callback_descriptor``. A PG-consumed task fires no
+ Celery ``.link``, so it claims its batch and runs the decrement in-body,
+ self-chaining the callback at ``remaining`` → 0.
"""
del app_instance # Protocol parity; callback dispatched by the decrement.
if not header_tasks:
@@ -604,19 +615,14 @@ class PgBarrier:
)
execution_id = str(execution_id)
- is_pg = is_pg_transport(transport)
try:
fairness_headers = fairness.as_header() if fairness else None
- callback_descriptor: CallbackDescriptor = {
- "task_name": callback_task_name,
- "kwargs": callback_kwargs,
- "queue": callback_queue,
- "fairness_headers": fairness_headers,
- }
- if is_pg:
- # The decrement (run in-body on the PG path) reads this to
- # self-chain the callback onto PG rather than Celery.
- callback_descriptor["transport"] = WorkflowTransport.PG_QUEUE.value
+ callback_descriptor = build_callback_descriptor(
+ task_name=callback_task_name,
+ kwargs=callback_kwargs,
+ queue=callback_queue,
+ fairness_headers=fairness_headers,
+ )
# expires_at = absolute orphan cap (6h). last_progress_at = now() (the
# reaper's fast stuck signal, re-stamped on every decrement).
ttl_seconds = barrier_ttl_seconds()
@@ -676,15 +682,13 @@ class PgBarrier:
self._dispatch_headers(
header_tasks,
- is_pg=is_pg,
execution_id=execution_id,
callback_descriptor=callback_descriptor,
fairness=fairness,
- fairness_headers=fairness_headers,
)
logger.info(
- f"Barrier enqueued via PgBarrier ({transport}) — "
+ f"Barrier enqueued via PgBarrier — "
f"exec_id={execution_id}, header_tasks={len(header_tasks)}, "
f"callback={callback_task_name}, queue={callback_queue}"
)
@@ -704,45 +708,28 @@ class PgBarrier:
self,
header_tasks: list[Signature],
*,
- is_pg: bool,
execution_id: str,
callback_descriptor: CallbackDescriptor,
fairness: FairnessKey | None,
- fairness_headers: dict[str, Any] | None,
) -> None:
- """Dispatch the N header tasks, one transport or the other.
-
- PG path (``is_pg``) → fire-and-forget onto the PG queue via
- :meth:`_dispatch_header_pg` (no ``.link``). Celery path → ``.link`` /
- ``.link_error`` chord-style. On any mid-loop dispatch failure, ``i`` of N
- never reached the queue so the counter can't reach 0 — delete the barrier
- row (and, on the PG path, reclaim dedup markers an earlier header may have
- committed, since the in-flight ``barrier_pg_abort`` is a no-op once the row
- is gone) so an in-flight decrement finds nothing, then re-raise.
+ """Dispatch the N header tasks fire-and-forget onto the PG queue.
+
+ On any mid-loop dispatch failure, ``i`` of N never reached the queue so
+ the counter can't reach 0 — delete the barrier row and reclaim the dedup
+ markers an earlier header may have committed (the in-flight
+ ``barrier_pg_abort`` is a no-op once the row is gone) so an in-flight
+ decrement finds nothing, then re-raise.
"""
- link_signature = barrier_pg_decr_and_check.s(
- execution_id=execution_id, callback_descriptor=callback_descriptor
- )
- link_error_signature = barrier_pg_abort.s(execution_id=execution_id)
for i, task in enumerate(header_tasks):
try:
- if is_pg:
- self._dispatch_header_pg(
- task, i, execution_id, callback_descriptor, fairness
- )
- else:
- cloned = task.clone()
- if fairness_headers:
- cloned.set(headers=fairness_headers)
- cloned.link(link_signature)
- cloned.link_error(link_error_signature)
- cloned.apply_async()
+ self._dispatch_header_pg(
+ task, i, execution_id, callback_descriptor, fairness
+ )
except Exception:
with contextlib.suppress(Exception):
_delete_barrier(execution_id)
- if is_pg:
- with contextlib.suppress(Exception):
- clear_execution_batches(execution_id)
+ with contextlib.suppress(Exception):
+ clear_execution_batches(execution_id)
logger.exception(
f"[exec:{execution_id}] header dispatch failed at task "
f"{i}/{len(header_tasks)}; barrier row deleted to prevent "
@@ -820,53 +807,28 @@ def _fire_barrier_callback(
) -> str:
"""Dispatch the aggregating callback when the barrier completes; return its id.
- Two transports, selected by the descriptor's ``transport`` marker:
-
- - ``"pg_queue"`` (fire-and-forget PG path): self-chain the callback onto
- the PG queue via :func:`_dispatch_pg`, carrying the producer's fairness
- (reconstructed from the stored headers) so the callback rides the same
- org/priority as the Celery path — no Celery, so the whole execution stays
- off the broker.
- - absent / anything else (legacy ``.link`` path): dispatch via
- ``current_app.signature(...).apply_async()`` — byte-identical to the legacy path,
- preserving the ``fairness_headers`` the producer attached.
-
- On the PG path the callback kwargs carry :data:`PG_TRANSPORT_CALLBACK_KWARG`
- so the aggregating callback can PG-gate its at-least-once duplicate guard (the
- callback is unguarded against the redelivery its own dispatch admits; the
- Celery ``.link`` path never injects it, so the guard is a no-op there).
- """
- if is_pg_transport(callback_descriptor.get("transport")):
- # Copy (don't mutate the shared descriptor) + tag the PG transport so the
- # callback can gate its duplicate guard on it.
- pg_kwargs = {
- **callback_descriptor["kwargs"],
- PG_TRANSPORT_CALLBACK_KWARG: True,
- }
- handle = _dispatch_pg(
- callback_descriptor["task_name"],
- args=[all_results],
- kwargs=pg_kwargs,
- queue=callback_descriptor["queue"],
- fairness=_fairness_from_headers(callback_descriptor.get("fairness_headers")),
- )
- return str(handle.id)
-
- from celery import current_app
+ Self-chains the callback onto the PG queue via :func:`_dispatch_pg`, carrying
+ the producer's fairness (reconstructed from the stored headers) so the
+ callback rides the same org/priority as its header tasks did.
- # Build the callback-signature kwargs explicitly (headers only when truthy) —
- # matching CeleryChordBarrier's idiom, no spurious headers=None.
- signature_kwargs: dict[str, Any] = {
- "args": [all_results],
- "kwargs": callback_descriptor["kwargs"],
- "queue": callback_descriptor["queue"],
+ The callback kwargs carry :data:`PG_TRANSPORT_CALLBACK_KWARG` so the
+ aggregating callback can gate its at-least-once duplicate guard — it is
+ otherwise unguarded against the redelivery its own dispatch admits.
+ """
+ # Copy (don't mutate the shared descriptor) + tag the transport so the
+ # callback can gate its duplicate guard on it.
+ pg_kwargs = {
+ **callback_descriptor["kwargs"],
+ PG_TRANSPORT_CALLBACK_KWARG: True,
}
- if callback_descriptor.get("fairness_headers"):
- signature_kwargs["headers"] = callback_descriptor["fairness_headers"]
- callback_signature = current_app.signature(
- callback_descriptor["task_name"], **signature_kwargs
+ handle = _dispatch_pg(
+ callback_descriptor["task_name"],
+ args=[all_results],
+ kwargs=pg_kwargs,
+ queue=callback_descriptor["queue"],
+ fairness=_fairness_from_headers(callback_descriptor.get("fairness_headers")),
)
- return str(callback_signature.apply_async().id)
+ return str(handle.id)
class _DecrementRow(NamedTuple):
@@ -907,12 +869,11 @@ def _apply_decrement(
that default at its source.)
- **COMMIT phase** — ``conn.commit()``. A failure here is AMBIGUOUS: the
server may have applied the commit before the socket dropped. Re-applying
- could double-count, so it is NEVER retried — it propagates. On the in-body
- PG path (:func:`run_batch_with_barrier`) the caller then tears the barrier
- down so the execution fails fast; on the Celery ``.link`` path
- (:func:`barrier_pg_decr_and_check`, ``max_retries=0``) it is not retried and
- the barrier is reclaimed at ``expires_at`` by the reaper. Either way the
- counter is never corrupted.
+ could double-count, so it is NEVER retried — it propagates. The in-body
+ caller (:func:`run_batch_with_barrier`) then tears the barrier down so the
+ execution fails fast, and a barrier left behind by a hard crash is
+ reclaimed at ``expires_at`` by the reaper. Either way the counter is never
+ corrupted.
Any non-connection error (e.g. the NUL-byte ``psycopg2.DataError`` from the
jsonb cast) also propagates unchanged — only the idle-reap self-heals. This is
@@ -1013,22 +974,22 @@ def _barrier_pg_decrement(
Appends this task's result and decrements ``remaining`` in one atomic
statement; the single caller that drives ``remaining`` to 0 dispatches the
- aggregating callback (then deletes the row). This is the plain, in-body-
- callable core shared by two entry points:
-
- - the Celery ``link`` callback :func:`barrier_pg_decr_and_check` (today's
- only caller, behaviour unchanged); and
- - the PG-consumed pipeline path, which will call this directly in
- the header task's body — a PG-consumed task fires no ``.link``, so the
- decrement must run in-body.
+ aggregating callback (then deletes the row). Called directly from the header
+ task's body by :func:`run_batch_with_barrier` — a PG-consumed task fires no
+ Celery ``.link``, so the decrement must run in-body. (It used to have a
+ second entry point, a ``@worker_task`` link wrapper, deleted with the Celery
+ transport in UN-4078.)
Callers MUST run each decrement in its own committed transaction and MUST NOT
replay a *committed* decrement: re-applying one that already landed corrupts
the count (fires the callback early with incomplete results, or skips past 0
and strands the barrier). This is enforced loudly, not just in prose — entry
raises if the shared connection is already mid-transaction (see the guard
- below), and the Celery wrapper pins ``max_retries=0`` so a task-level replay
- can't re-drive it; an orphaned barrier is bounded by ``expires_at`` instead.
+ below). The task-level replay guard that used to back this up — the Celery
+ wrapper's ``max_retries=0`` — went with ``barrier_pg_decr_and_check`` in
+ UN-4078; the decrement now runs in-body, so a replay would be a redelivery of
+ the batch itself, which ``claim_batch`` already makes idempotent. An orphaned
+ barrier is bounded by ``expires_at``.
The decrement DOES self-heal one narrow, provably-safe case via
:func:`_apply_decrement`: an execute-phase failure on a cached connection that
@@ -1163,25 +1124,6 @@ def _barrier_pg_decrement(
}
-@worker_task(name="barrier_pg_decr_and_check", max_retries=0)
-def barrier_pg_decr_and_check(
- result: Any,
- *,
- execution_id: str,
- callback_descriptor: CallbackDescriptor,
-) -> dict[str, Any]:
- """Per-task ``link`` callback for :class:`PgBarrier` (Celery entry point).
-
- Thin ``@worker_task`` wrapper around :func:`_barrier_pg_decrement` so the
- decrement logic stays callable in-body (no ``.link``) on the PG-consumed
- pipeline path. ``max_retries=0`` — a Celery retry would replay
- the decrement and corrupt the count (see the core's contract).
- """
- return _barrier_pg_decrement(
- result, execution_id=execution_id, callback_descriptor=callback_descriptor
- )
-
-
@worker_task(name="barrier_pg_abort", max_retries=0)
def barrier_pg_abort(
request: Any = None,
diff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.py
index b43438c44..9d9c41cd8 100644
--- a/workers/queue_backend/pg_queue/reaper.py
+++ b/workers/queue_backend/pg_queue/reaper.py
@@ -46,10 +46,14 @@ standby. A standby tries to acquire each cycle. The cycle interval MUST be
shorter than the lease window, or the leader would lose the lease between
renews — enforced in :meth:`PgReaper.__init__`.
-**Ships dark.** Launched explicitly (``python -m queue_backend.pg_queue.reaper``
-or, later, ``run-worker.sh``); never part of the default worker set. With
-``WORKER_BARRIER_BACKEND`` left at ``chord`` (default) there are no
-``pg_barrier_state`` rows, so the sweep is a no-op until the PG barrier is used.
+**A hard dependency, not an optional sweep.** This used to ship dark: the PG
+barrier was one of three selectable substrates, so with ``WORKER_BARRIER_BACKEND``
+at its ``chord`` default there were no ``pg_barrier_state`` rows and nothing to
+reap. That is no longer true. PG is the only transport and the only barrier, and
+this process additionally owns queue re-arm (crash redelivery) and scheduled
+promotion (delayed delivery) — if it is not running, crashed work is never
+redelivered and delayed messages never fire. Deployed as ``worker-pg-reaper``;
+alert on its liveness.
"""
from __future__ import annotations
diff --git a/workers/queue_backend/redis_barrier.py b/workers/queue_backend/redis_barrier.py
deleted file mode 100644
index 2c1e2fb3e..000000000
--- a/workers/queue_backend/redis_barrier.py
+++ /dev/null
@@ -1,761 +0,0 @@
-"""Redis-DECR Barrier — alternative substrate to ``CeleryChordBarrier``.
-
-Implements the ``Barrier`` Protocol via the labs-design distributed-counter
-pattern (``DECR remaining:{exec_id}``) with per-task result aggregation
-in a Redis list. Selected at runtime by the ``WORKER_BARRIER_BACKEND``
-env flag (default ``chord`` — see ``queue_backend.get_barrier``).
-
-**Why this substrate exists.** Celery's ``chord(header)(body)`` aggregator
-is the highest-risk Celery construct at our scale — documented silent
-task drops at ~130K-task scale (see PG Queue decision journey). This
-substrate replaces *only* the aggregation primitive; everything else
-(dispatch, retries, result backend, monitoring) still rides on Celery.
-The transport is unchanged — only the "wait for N tasks to complete,
-then fire the callback with their results" coordination is moved from
-Celery's chord backend to Redis.
-
-**Wire model.**
-
-1. ``enqueue``: ``SET remaining:{exec_id} N`` with TTL (default 6h,
- tunable via ``WORKER_BARRIER_KEY_TTL_SECONDS``). The results list
- is created lazily by the first link task's ``RPUSH`` — its TTL is
- set inside the Lua script after the push, so both keys share the
- same expiry window. Each header task is dispatched with
- ``.link(barrier_decr_and_check.s(...))`` (success) and
- ``.link_error(barrier_abort.s(...))`` (failure).
-2. Per-task success: the link task runs ``RPUSH results + EXPIRE +
- DECR remaining`` atomically via a Lua script. If the post-decrement
- counter reads exactly 0, the link reads the full results list,
- dispatches the callback with it as the first arg, then deletes
- the Redis keys.
-3. Per-task failure: ``barrier_abort`` runs as a link_error, DELing
- both barrier keys + logging the failure. Mirrors Celery chord's
- default error-propagation semantic (callback isn't invoked when
- any header fails). The outer task's error handler marks the
- workflow FAILED; this task only cleans barrier state.
-4. TTL safety net: if the link/link_error never fire for some task
- (worker crash mid-execution, broker outage), TTL expires the keys
- after the configured window. Subsequent successful tasks' DECR on
- the missing counter returns negative; the Lua script's ``< 0``
- branch DELs cleanly without firing a spurious callback.
-
-**Result-aggregation parity.** The callback receives
-``list[BatchExecutionResult]`` exactly as it does today under
-``CeleryChordBarrier`` — zero callback-side changes. Serialisation
-rides through ``BatchExecutionResult.to_dict()`` / ``from_dict()``
-(the typed callback boundary) ↔ JSON in Redis.
-
-**Atomicity.** The per-task ``RPUSH + DECR`` is atomic via Lua — a
-network partition mid-step can't split-brain the counter and the
-result list. ``DECR`` is also atomic on its own, so exactly one task
-sees the post-decrement 0.
-
-**Mixed-version rolling deploy.** The ``WORKER_BARRIER_BACKEND`` env
-var is read at worker startup, so each pod independently picks its
-substrate. A given execution's tasks all share the publishing pod's
-substrate choice (the barrier is selected once per execution start);
-no cross-substrate aggregation within a single execution.
-"""
-
-from __future__ import annotations
-
-import contextlib
-import json
-import logging
-import os
-from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any
-
-from celery.canvas import Signature
-
-from unstract.core.cache.redis_client import create_redis_client
-from unstract.core.data_models import DEFAULT_WORKFLOW_TRANSPORT
-
-from .barrier import (
- _DEFAULT_BARRIER_TTL_SECONDS,
- CallbackDescriptor,
- barrier_ttl_seconds,
-)
-from .decorator import worker_task
-from .fairness import FairnessKey
-from .handle import BarrierHandle
-
-if TYPE_CHECKING:
- import redis as redis_lib
-
-logger = logging.getLogger(__name__)
-
-
-# Env-var prefix for the barrier's Redis connection. Falls back to
-# ``REDIS_`` (the canonical worker Redis instance) via ``create_redis_client``'s
-# nested-getenv pattern. Operators can point the barrier at a separate
-# Redis by setting ``WORKER_BARRIER_REDIS_HOST`` / ``_PORT`` etc.
-_BARRIER_REDIS_ENV_PREFIX = "WORKER_BARRIER_REDIS_"
-
-# Belt-and-suspenders cleanup for orphaned ``remaining`` / ``results``
-# keys. On the happy path the Lua script ``DEL``s both keys when
-# ``remaining`` reaches 0 — TTL only matters when the counter stalls
-# (e.g. some header tasks fail and the link never fires for them).
-#
-# **Must be strictly > longest plausible execution wall-clock.** If
-# TTL expires while tasks are still pending, the next successful
-# task's ``DECR`` creates the counter at ``-1`` and the script fires
-# the callback with just that task's result — followed by more
-# spurious callbacks as remaining tasks complete.
-#
-# Reference budget: the operator-tuned ceiling is whatever
-# ``FILE_PROCESSING_TASK_TIME_LIMIT`` is set to in the deployment env
-# (workers/sample.env ships 7200/2h; docker/sample.env up to 10800/3h).
-# ``process_file_batch`` has ``max_retries=0`` so no retry amplification —
-# worst case per batch is the time-limit ceiling. Multi-batch
-# executions ≈ max(per-batch time) since batches run in parallel
-# (worker concurrency is the limiter, not serial wait).
-#
-# 6h default gives 2-3x margin over the documented per-batch ceiling
-# while staying in the same order of magnitude as
-# ``FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000`` (5h). Operators with
-# longer workflows (e.g. multi-step pipelines with chained barriers)
-# or shorter known max-execution-time should tune via
-# ``WORKER_BARRIER_KEY_TTL_SECONDS``.
-# TTL lives in barrier.py (shared with PgBarrier — both bound an orphaned
-# barrier by the same env var). Aliased so this module's internal call sites and
-# tests keep their names.
-_key_ttl_seconds = barrier_ttl_seconds
-_KEY_TTL_DEFAULT_SECONDS = _DEFAULT_BARRIER_TTL_SECONDS
-
-
-# Atomic ``RPUSH + EXPIRE + DECR``: returns ``(remaining, results)``.
-#
-# Three branches the Python side reads off the returned counter:
-#
-# remaining > 0 — pending, no results returned (still aggregating)
-# remaining == 0 — complete, all results returned (we are the last).
-# Keys are NOT DEL'd here — Python defers the DEL
-# until after ``apply_async()`` succeeds, so that
-# a callback-dispatch failure leaves the keys (and
-# their TTL) in place rather than stranding the
-# execution with no state + no recovery path.
-# remaining < 0 — TTL-expired counter (or replay after cleanup);
-# keys DEL'd inside the script to prevent further
-# spurious fires, no callback dispatched.
-#
-# Setting the ``results`` key TTL inside the script — ``EXPIRE`` on
-# the call AFTER ``RPUSH`` ensures the key exists when EXPIRE runs
-# (issuing EXPIRE before any push is a no-op on a non-existent key,
-# which is the bug the previous version had). Repeated EXPIRE per
-# RPUSH is cheap and keeps both keys' TTLs aligned.
-_RPUSH_DECR_LUA = """
-local remaining_key = KEYS[1]
-local results_key = KEYS[2]
-local result_json = ARGV[1]
-local ttl_seconds = tonumber(ARGV[2])
-redis.call("RPUSH", results_key, result_json)
-redis.call("EXPIRE", results_key, ttl_seconds)
-local remaining = redis.call("DECR", remaining_key)
-if remaining == 0 then
- -- We're the last task. LRANGE the results for the Python side to
- -- dispatch the callback. DO NOT DEL the keys here — Python defers
- -- the DEL until after apply_async() succeeds so a dispatch failure
- -- (broker outage, serialisation error) leaves the keys + TTL in
- -- place. Without this deferral, an apply_async failure would
- -- strand the execution with both keys gone, no TTL, no Celery
- -- retry (max_retries=0), and no link_error fallback — strictly
- -- worse than the chord baseline (where Celery's chord backend
- -- owns + retries body invocation).
- local all_results = redis.call("LRANGE", results_key, 0, -1)
- return {remaining, all_results}
-end
-if remaining < 0 then
- -- Counter was TTL-expired (or already cleaned up) when this task
- -- ran. ``DECR`` on a missing key created it at ``-1``; subsequent
- -- tasks would land here too. DEL both keys so neither this task
- -- nor any subsequent one fires a spurious callback; the abandoned
- -- branch on the Python side surfaces an error-severity log.
- redis.call("DEL", remaining_key, results_key)
- return {remaining, {}}
-end
-return {remaining, {}}
-"""
-
-
-# Process-local cached Redis client. Lazy-initialised on first call
-# within a worker process; reused across every ``barrier_decr_and_check``
-# / ``barrier_abort`` / ``RedisDecrBarrier`` invocation in that process.
-#
-# Why a singleton: ``create_redis_client`` builds a fresh
-# ``ConnectionPool`` each call. Without caching, a 1000-task fan-out
-# would spin up 1000 separate pools (1000 TCP handshakes torn down in
-# sequence) — exactly the scale this substrate exists to handle.
-#
-# Fork safety: Celery's prefork model re-imports this module in each
-# child process, so each worker process gets its own fresh
-# ``_redis_client_singleton`` — no socket sharing across fork
-# boundaries (which would corrupt state).
-#
-# Thread safety: redis-py's ``ConnectionPool`` is internally thread-safe,
-# so concurrent users of the cached client are fine. The lazy-init
-# itself is not locked — two threads racing on the first call could
-# briefly create two pools, with one being orphaned. The cost is one
-# short-lived pool (immediately GC'd), not a correctness issue.
-_redis_client_singleton: redis_lib.Redis | None = None
-
-
-def _get_redis_client() -> redis_lib.Redis:
- """Return a process-cached Redis client for the barrier.
-
- ``create_redis_client``'s nested-getenv fallback only covers
- HOST/PORT/PASSWORD/USER/DB — it does NOT cross-fall-back
- SENTINEL_MODE or SSL. In a deployment where the canonical Redis
- is Sentinel-backed or TLS-secured, leaving ``WORKER_BARRIER_REDIS_*``
- unset and relying on the documented HOST fallback would result in
- the barrier inheriting the right host/port but connecting
- standalone/plaintext — which fails (or worse, connects without
- TLS) every execution the moment the flag flips.
-
- To avoid that footgun: when no barrier-specific HOST is set, use
- the canonical ``REDIS_`` prefix *directly* so we inherit the FULL
- canonical config (including Sentinel + SSL). When an operator
- has set ``WORKER_BARRIER_REDIS_HOST`` they've opted into a
- dedicated barrier Redis and are responsible for the full config
- (HOST + PORT + ... + SENTINEL_MODE + SSL).
-
- ``decode_responses=True`` so ``LRANGE`` returns ``list[str]``
- (we JSON-decode each entry).
- """
- global _redis_client_singleton
- if _redis_client_singleton is None:
- # If no dedicated barrier-Redis host is configured, use the
- # canonical REDIS_ prefix so Sentinel/SSL config inherits too.
- if os.getenv(f"{_BARRIER_REDIS_ENV_PREFIX}HOST"):
- env_prefix = _BARRIER_REDIS_ENV_PREFIX
- else:
- env_prefix = "REDIS_"
- _redis_client_singleton = create_redis_client(
- env_prefix=env_prefix,
- decode_responses=True,
- socket_timeout=5,
- socket_connect_timeout=5,
- )
- return _redis_client_singleton
-
-
-def _remaining_key(execution_id: str) -> str:
- return f"barrier:remaining:{execution_id}"
-
-
-def _results_key(execution_id: str) -> str:
- return f"barrier:results:{execution_id}"
-
-
-def _abort_lock_key(execution_id: str) -> str:
- """Per-execution dedup lock for ``barrier_abort``.
-
- Set via ``SET NX EX`` in ``barrier_abort`` to collapse N concurrent
- header-task failures into a single Sentry/error event. MUST be
- cleared at the start of a new barrier (in ``enqueue``) — otherwise
- a retry that reuses an ``execution_id`` would hit a stale lock,
- early-exit the abort path, and silently mask the failed retry as
- successful.
- """
- return f"barrier:abort_lock:{execution_id}"
-
-
-class RedisDecrBarrier:
- """``Barrier`` implementation via Redis ``DECR remaining`` pattern.
-
- Drop-in replacement for ``CeleryChordBarrier`` from the call
- sites' perspective — same ``enqueue(header_tasks, ...)`` signature,
- same ``BarrierHandle | None`` return contract, same fairness
- plumbing. The substrate difference is invisible above the
- ``Barrier`` Protocol.
-
- See module docstring for the wire model, atomicity, and failure
- semantics.
- """
-
- def __init__(self, redis_client: redis_lib.Redis | None = None) -> None:
- """Args:
- redis_client: Optional pre-built client. Tests pass a fake
- (``fakeredis``) here. Production leaves it ``None`` so
- the barrier builds its own from env on first use.
- """
- self._redis_client = redis_client
-
- @property
- def redis(self) -> redis_lib.Redis:
- """Lazy-init the Redis client.
-
- Built on first use rather than at construction time so the
- module can be imported (and the worker task registered) in
- contexts where Redis isn't yet reachable (e.g. test collection,
- worker boot before networking up).
-
- Production path: ``self._redis_client is None`` → falls back to
- (and caches) the module-level ``_redis_client_singleton``, so
- the instance and the standalone ``barrier_decr_and_check`` /
- ``barrier_abort`` worker tasks all share the same
- ``ConnectionPool``.
-
- Test path: ``__init__(redis_client=...)`` bypasses this branch
- entirely — fixtures inject a ``MagicMock`` / ``fakeredis``
- without touching the module-level singleton (which would leak
- across tests).
- """
- if self._redis_client is None:
- self._redis_client = _get_redis_client()
- return self._redis_client
-
- def enqueue(
- self,
- header_tasks: list[Signature],
- *,
- callback_task_name: str,
- callback_kwargs: dict[str, Any],
- callback_queue: str,
- app_instance: Any,
- fairness: FairnessKey | None = None,
- transport: str = DEFAULT_WORKFLOW_TRANSPORT,
- ) -> BarrierHandle | None:
- """See :class:`queue_backend.barrier.Barrier.enqueue`.
-
- ``transport`` is accepted for Protocol parity and ignored — the Redis
- barrier is the Celery-transport fan-in substrate (the PG transport uses
- ``PgBarrier``'s fire-and-forget mode). Mirrors ``CeleryChordBarrier``:
-
- - Empty ``header_tasks`` → ``None`` (caller handles the
- zero-files contract); no Redis keys touched.
- - Any substrate-level failure (Redis down, dispatch error,
- serialisation issue) → raises.
- - Success → returns a ``BarrierHandle`` whose ``.id`` is the
- execution id (no ``AsyncResult`` since there's no Celery
- chord wrapping the group; the id is what the call site
- logs for chord-id traceability).
-
- Note: ``app_instance`` is accepted for ``Barrier`` Protocol
- parity with ``CeleryChordBarrier`` but unused by this
- substrate. The callback signature is built inside the link
- task via ``celery.current_app.signature(...)`` (so the link
- runs against the *worker's* app, not the producer's). Keeping
- the parameter in the signature lets call sites swap between
- substrates without per-substrate adapter code.
- """
- # Explicit ``del`` documents the Protocol-parity intent and
- # satisfies static linters' unused-parameter check.
- del app_instance, transport
- if not header_tasks:
- execution_id = callback_kwargs.get("execution_id")
- pipeline_id = callback_kwargs.get("pipeline_id")
- logger.info(
- f"[exec:{execution_id}] [pipeline:{pipeline_id}] "
- "Zero header tasks detected — skipping barrier enqueue "
- "(parent should handle pipeline status updates directly)"
- )
- return None
-
- execution_id = callback_kwargs.get("execution_id")
- if not execution_id:
- raise ValueError(
- "RedisDecrBarrier requires execution_id in callback_kwargs — "
- "it's the key suffix for the per-execution remaining/results "
- "Redis keys"
- )
- execution_id = str(execution_id)
-
- try:
- fairness_headers = fairness.as_header() if fairness else None
-
- # The link task body needs to know:
- # - which execution's counter to decrement
- # - how to reconstruct + dispatch the callback when count hits 0
- # All Celery-serialisable so the link can run on any worker.
- callback_descriptor: CallbackDescriptor = {
- "task_name": callback_task_name,
- "kwargs": callback_kwargs,
- "queue": callback_queue,
- "fairness_headers": fairness_headers,
- }
-
- # Initialise the counter before any header task can fire.
- # ``SET remaining N EX <ttl>`` overwrites any stale counter
- # left from a prior run with the same exec_id; the upfront
- # ``DEL results, abort_lock`` (below) clears the rest of
- # the prior-run state (the results list grows via RPUSH so
- # we must clear it explicitly; the abort_lock would otherwise
- # silently mask a retry's failure as success — see the
- # comment block at the DELETE call). ``ttl`` is
- # ``_key_ttl_seconds()`` (6h default).
- ttl_seconds = _key_ttl_seconds()
- # Clear any leftover state from a prior execution with this
- # execution_id (including the dedup lock written by a
- # previous run's ``barrier_abort``). Without DELing the
- # abort_lock here, a retry that reuses execution_id would
- # find the stale lock and route every ``barrier_abort``
- # straight to the deduplicated branch — leaving B's
- # ``remaining``/``results`` keys uncleaned. In-flight
- # successful tasks from B would then hit ``remaining == 0``
- # via DECR and fire the callback with partial results,
- # silently masking the failed retry as successful.
- self.redis.delete(
- _results_key(execution_id),
- _abort_lock_key(execution_id),
- )
- self.redis.set(
- _remaining_key(execution_id),
- len(header_tasks),
- ex=ttl_seconds,
- )
- # ``results`` key TTL is set inside the Lua script on
- # first RPUSH — ``EXPIRE`` on a non-existent key is a
- # no-op, so we can't pre-set it here.
-
- # Stamp fairness on each header task and attach the link
- # (success) + link_error (failure) callbacks.
- # ``Signature.clone()`` avoids mutating the caller's list
- # (same reasoning as ``CeleryChordBarrier``).
- link_signature = barrier_decr_and_check.s(
- execution_id=execution_id,
- callback_descriptor=callback_descriptor,
- )
- # link_error explicitly propagates header-task failures —
- # mirrors Celery chord's default error semantic (chord
- # callback isn't invoked on header failure). Without this,
- # a failed header task would leave the counter stuck and
- # the execution would hang until TTL cleanup. The abort
- # task DELs the barrier keys + logs the error; the outer
- # task's error path is responsible for the workflow
- # status update.
- link_error_signature = barrier_abort.s(execution_id=execution_id)
- for i, task in enumerate(header_tasks):
- try:
- cloned = task.clone()
- if fairness_headers:
- cloned.set(headers=fairness_headers)
- cloned.link(link_signature)
- cloned.link_error(link_error_signature)
- cloned.apply_async()
- except Exception:
- # Mid-loop dispatch failure: tasks 0..i-1 are
- # already in flight with the link attached, but
- # ``i`` of N never made it to the broker. The
- # counter would never reach 0 → execution hangs.
- # DEL the barrier keys so the in-flight links'
- # DECR-on-missing returns negative and the Lua
- # script's ``< 0`` branch DELs cleanly. Re-raise
- # so the caller's status-update path marks the
- # workflow ERROR.
- self.redis.delete(
- _remaining_key(execution_id),
- _results_key(execution_id),
- )
- # ``logger.exception`` captures the underlying
- # apply_async failure cause (broker timeout vs
- # serialisation error vs routing error) alongside
- # the orphan-count context — without it, a reader
- # of this log can't tell why dispatch failed and
- # has to cross-reference the outer handler's log.
- logger.exception(
- f"[exec:{execution_id}] apply_async failed at "
- f"task {i}/{len(header_tasks)}; {i} orphan tasks "
- f"already dispatched. Barrier keys DEL'd to "
- f"prevent spurious callback fires from the "
- f"orphan tasks' link decrements."
- )
- raise
-
- logger.info(
- f"Barrier enqueued via RedisDecrBarrier — "
- f"exec_id={execution_id}, "
- f"header_tasks={len(header_tasks)}, "
- f"callback={callback_task_name}, "
- f"queue={callback_queue}"
- )
-
- # Handle's ``.id`` is the execution id — same call sites
- # log ``chord_id`` from this; the semantic shifts from
- # "Celery chord aggregator task id" to "execution id" but
- # both are stable per-execution identifiers and the log
- # consumers don't depend on the underlying task vs exec id
- # distinction.
- return _RedisBarrierHandle(id=execution_id)
-
- except Exception:
- execution_id = callback_kwargs.get("execution_id")
- pipeline_id = callback_kwargs.get("pipeline_id")
- logger.exception(
- f"[exec:{execution_id}] [pipeline:{pipeline_id}] "
- f"Failed to enqueue barrier via Redis "
- f"(callback={callback_task_name}, queue={callback_queue}, "
- f"header_tasks={len(header_tasks)})"
- )
- raise
-
-
-@dataclass(frozen=True, slots=True)
-class _RedisBarrierHandle:
- """Minimal ``BarrierHandle`` implementation.
-
- Just an ``id`` attribute (the execution id) so call sites that
- log ``handle.id`` for chord-id tracing keep working. A class
- (rather than a ``namedtuple``) gives a clear home for the
- docstring + ``slots=True`` memory control; the ``BarrierHandle``
- Protocol is structural so any object exposing ``id: str`` would
- type-check.
- """
-
- id: str
-
-
-@worker_task(
- name="barrier_decr_and_check",
- # ``max_retries=0`` — the link task is intentionally non-idempotent
- # at the Redis-script level (RPUSH + DECR are committed atomically,
- # but a Celery retry would replay the script and double-count). Any
- # Celery-level retry would cause a second RPUSH + DECR, corrupting
- # the aggregation. If the link fails after the Lua commit (e.g. the
- # subsequent callback ``apply_async`` raises), the counter has
- # already advanced — TTL-driven cleanup is the safety net, and the
- # outer execution's error path marks the workflow status.
- #
- # Trade-off: a transient Redis blip on the link's Lua call means
- # this task's contribution is lost (counter doesn't advance) and
- # the execution hangs until TTL. Acceptable because (a) the
- # alternative — letting Celery retry — corrupts the aggregation,
- # and (b) Redis blips are operator-visible via monitoring, unlike
- # silent double-counts.
- max_retries=0,
-)
-def barrier_decr_and_check(
- result: Any,
- *,
- execution_id: str,
- callback_descriptor: CallbackDescriptor,
-) -> dict[str, Any]:
- """Per-task link callback for ``RedisDecrBarrier``.
-
- Runs after each header task completes successfully. Pushes the
- task's result into the per-execution Redis list, decrements the
- counter, and (if the post-decrement counter reads 0) dispatches
- the aggregating callback with the full result list.
-
- ``result`` is whatever the header task returned — for ``process_file_batch``
- that's a ``BatchExecutionResult`` already serialised to a dict via
- the typed callback boundary. We pass it through ``json.dumps`` to
- survive the Redis round-trip.
- """
- from celery import current_app
-
- try:
- # No ``default=str`` — that would silently stringify any non-
- # JSON-safe leaf (datetime, UUID, Decimal, sets...) and mask
- # the typed-boundary regression the surrounding ``try/except``
- # is here to surface. The header task's return value is
- # expected to be a fully JSON-safe dict per the
- # ``BatchExecutionResult.to_dict()`` contract; anything else
- # should fail loudly here so the regression surfaces.
- result_json = json.dumps(result)
- except (TypeError, ValueError):
- logger.exception(
- f"[exec:{execution_id}] Header task result is not "
- f"JSON-serialisable — barrier aggregation cannot proceed. "
- f"This indicates a typed-boundary regression; the "
- f"BatchExecutionResult.to_dict() contract must produce a "
- f"JSON-safe shape."
- )
- raise
-
- redis_client = _get_redis_client()
- script = redis_client.register_script(_RPUSH_DECR_LUA)
- remaining, raw_results = script(
- keys=[
- _remaining_key(execution_id),
- _results_key(execution_id),
- ],
- args=[result_json, _key_ttl_seconds()],
- )
-
- logger.info(f"[exec:{execution_id}] Barrier link DECR → remaining={remaining}")
-
- if remaining > 0:
- # More tasks pending — nothing to do.
- return {"status": "pending", "remaining": remaining}
-
- if remaining < 0:
- # TTL-expired counter (or replay after cleanup). The Lua
- # script already DEL'd both keys to prevent further spurious
- # fires on subsequent task completions. No callback dispatch.
- #
- # Logged at ERROR (not WARNING) because reaching this branch
- # means the barrier was torn down out from under in-flight
- # tasks — by definition an abnormal terminal state where the
- # execution almost certainly did not complete normally.
- # Returning normally from a .link is indistinguishable from
- # success to Celery; the ERROR log is the only execution-id-
- # tagged signal an operator can correlate to the hung/
- # incorrectly-statused execution.
- logger.error(
- f"[exec:{execution_id}] Barrier abandoned — counter went "
- f"negative (remaining={remaining}); keys were TTL-expired "
- f"or torn down. No callback dispatched; execution likely "
- f"in an inconsistent terminal state and needs investigation."
- )
- return {"status": "abandoned", "remaining": remaining}
-
- # remaining == 0: we're the last task. Dispatch the callback.
- all_results = [json.loads(r) for r in raw_results]
- callback_task_name = callback_descriptor["task_name"]
- callback_kwargs = callback_descriptor["kwargs"]
- callback_queue = callback_descriptor["queue"]
- fairness_headers = callback_descriptor.get("fairness_headers")
-
- callback_signature = current_app.signature(
- callback_task_name,
- args=[all_results],
- kwargs=callback_kwargs,
- queue=callback_queue,
- **({"headers": fairness_headers} if fairness_headers else {}),
- )
- # Dispatch FIRST; DEL the barrier keys only after dispatch succeeds.
- #
- # If apply_async raises (broker outage, serialisation error,
- # routing failure), the exception propagates. The barrier keys
- # are NOT yet DEL'd, so:
- # - Subsequent late-arriving link tasks (if any) hit the Lua
- # ``< 0`` branch (counter is still 0 — DECR-on-zero would
- # yield -1) and exit cleanly via the abandoned branch.
- # - The keys TTL-expire eventually — Redis cleans them up.
- # - The link task's failure surfaces via Celery's standard
- # task-failure channels.
- #
- # If we DEL'd before apply_async (the previous design), an
- # apply_async failure would strand the execution with no keys,
- # no TTL, no Celery retry (max_retries=0), and no link_error
- # fallback — silent infinite hang. The deferred DEL is what
- # restores parity-or-better with the chord baseline.
- callback_result = callback_signature.apply_async()
- redis_client.delete(
- _remaining_key(execution_id),
- _results_key(execution_id),
- )
-
- logger.info(
- f"[exec:{execution_id}] Barrier complete — "
- f"fired callback {callback_task_name} on {callback_queue} "
- f"with {len(all_results)} aggregated results "
- f"(callback_task_id={callback_result.id})"
- )
-
- return {
- "status": "complete",
- "callback_task_id": callback_result.id,
- "aggregated_count": len(all_results),
- }
-
-
-@worker_task(
- name="barrier_abort",
- # Same posture as ``barrier_decr_and_check`` — non-idempotent at
- # the Redis level (DEL is fine to repeat, but we don't want
- # double-logged abort events polluting Sentry).
- max_retries=0,
-)
-def barrier_abort(
- request: Any = None,
- exc: Any = None,
- traceback: Any = None,
- *,
- execution_id: str,
-) -> dict[str, Any]:
- """``link_error`` callback: header task failed → clean up barrier state.
-
- Celery 5.5 invokes new-style errbacks as
- ``errback(request, exc, traceback)`` only for ``bind=False`` tasks
- with arity > 1. If a worker hits the ``NotRegistered`` fallback
- during a mixed-version rolling deploy (or if this task is ever
- switched to ``bind=True``), Celery calls the errback old-style
- with just ``(task_id,)``. The defaults on ``request`` / ``exc`` /
- ``traceback`` mean the old-style path degrades to a clean
- ``exc=None`` log line rather than a confusing
- ``TypeError: missing required positional arguments``.
-
- Mirrors Celery chord's default error semantic: when any header
- task fails, the chord callback isn't invoked. Without explicit
- cleanup the ``remaining`` counter would stay above 0 until TTL
- expiry, the ``results`` key would leak, and any subsequent
- successful header tasks' link would still DECR — eventually
- spurious-firing the callback with partial results (or via the
- Lua ``< 0`` branch, abandoning the execution but only after the
- extra task work).
-
- We DEL the barrier keys here so the in-flight successful tasks'
- link DECR returns negative → Lua ``< 0`` branch fires →
- aggregation cleanly abandoned. **Terminal status drivers**: this
- task does NOT mark the workflow FAILED — the outer orchestrators
- (``_run_workflow_api`` / ``_orchestrate_file_processing_general``)
- wrap only the *synchronous* fan-out in try/except and have
- already returned by the time ``barrier_abort`` runs. Workflow
- terminal status on async header failure is driven by per-file
- updates inside ``process_file_batch`` plus the eventual
- aggregating callback (which won't fire because we DEL'd the
- keys). A future Phase 6b' refinement may have ``barrier_abort``
- itself drive the execution-level status update.
-
- **Concurrent-failure dedup.** Every header task attaches the same
- ``link_error``, so N simultaneous failures would otherwise fire N
- ``barrier_abort`` executions — each calling ``logger.error`` and
- each producing its own Sentry event for what is a *single* logical
- execution failure. A ``SET NX`` lock on
- ``barrier:abort_lock:{execution_id}`` makes the first abort win;
- subsequent aborts for the same execution see the lock present and
- early-exit silently. The DELs are idempotent anyway (DEL on a
- missing key is a no-op), but the dedup collapses the alert noise.
-
- **Lock release on DELETE failure.** If SET NX succeeds (this
- abort wins) but the subsequent DELETE then raises (mid-abort
- Redis blip), the lock stays held while the barrier keys are NOT
- deleted. Every sibling ``barrier_abort`` would then hit the
- dedup early-exit and do nothing, leaving the counter alive for
- in-flight successful tasks to DECR to 0 → callback fires with
- partial results → silent success masking. To close this hole,
- we explicitly release the abort_lock if the DELETE raises, so
- a sibling abort can re-acquire it and complete the cleanup.
- """
- redis_client = _get_redis_client()
- # ``SET NX EX`` — first-write-wins with auto-expiry. TTL matches
- # the barrier keys' TTL so the lock cleans up alongside them.
- # The lock is also explicitly DEL'd by ``enqueue`` at the start
- # of any new execution with the same execution_id, to prevent a
- # stale lock from masking a retry's failure as success.
- if not redis_client.set(
- _abort_lock_key(execution_id), "1", ex=_key_ttl_seconds(), nx=True
- ):
- # Another barrier_abort for this execution already won the
- # SET NX race. Silently exit — no log, no Sentry, no
- # duplicate DEL.
- return {
- "status": "deduplicated",
- "execution_id": execution_id,
- }
- try:
- redis_client.delete(
- _remaining_key(execution_id),
- _results_key(execution_id),
- )
- except Exception:
- # DELETE failed AFTER we won the SET NX. Release the lock so
- # subsequent aborts aren't dedup-suppressed into no-ops while
- # the barrier keys still exist. ``suppress`` because if the
- # lock-release also fails, the original DELETE exception is
- # the more useful one to propagate (Redis is clearly in
- # trouble and the original error captures that).
- with contextlib.suppress(Exception):
- redis_client.delete(_abort_lock_key(execution_id))
- raise
- logger.error(
- f"[exec:{execution_id}] Header task failed; barrier aborted. "
- f"Cleaned up remaining/results keys. exc={exc!r}"
- )
- return {
- "status": "aborted",
- "execution_id": execution_id,
- "reason": str(exc),
- }
diff --git a/workers/scheduler/tasks.py b/workers/scheduler/tasks.py
index b30742c45..d79f7bf75 100644
--- a/workers/scheduler/tasks.py
+++ b/workers/scheduler/tasks.py
@@ -39,11 +39,10 @@ from shared.patterns.notification.helper import trigger_notification
from shared.utils.api_client_singleton import get_singleton_api_client
from unstract.core.data_models import (
- DEFAULT_WORKFLOW_TRANSPORT,
+ LEGACY_TRANSPORT_KEY,
+ LEGACY_TRANSPORT_VALUE,
NotificationPayload,
NotificationSource,
- is_pg_transport,
- normalize_transport,
)
# Import the exact backend logic to ensure consistency
@@ -159,22 +158,11 @@ def _execute_scheduled_workflow(
pipeline_id=context.pipeline_id,
)
- # Transport this execution rides (9e), decided by the backend at creation
- # and carried in the dispatched task's payload. Absent key (older backend)
- # → celery default; a present-but-unrecognized value (version skew) is
- # coerced to celery with a loud warning rather than dispatched onto an
- # unknown substrate (fail-closed).
- transport = normalize_transport(
- workflow_execution.get("transport", DEFAULT_WORKFLOW_TRANSPORT),
- logger=logger,
- context=f" [exec:{execution_id}]",
- )
-
logger.info(
f"[exec:{execution_id}] [pipeline:{context.pipeline_id}] Created workflow execution for scheduled pipeline {context.pipeline_name}"
)
- # Step 2: Trigger async workflow execution via direct Celery dispatch
+ # Step 2: Trigger async workflow execution
logger.info(
f"[exec:{execution_id}] [pipeline:{context.pipeline_id}] Triggering async execution for workflow {context.workflow_id}"
)
@@ -196,13 +184,13 @@ def _execute_scheduled_workflow(
kwargs={
"use_file_history": context.use_file_history,
"pipeline_id": context.pipeline_id,
- "transport": transport,
+ # Rolling-deploy shim: see LEGACY_TRANSPORT_KEY.
+ LEGACY_TRANSPORT_KEY: LEGACY_TRANSPORT_VALUE,
},
queue=QueueName.GENERAL,
- # Orchestrator transport (9e PR A / 2d): route async_execute_bin
- # onto PG for a pg_queue execution (carried-marker wins over the
- # allow-list); None keeps the legacy Celery dispatch.
- backend=QueueBackend.PG if is_pg_transport(transport) else None,
+ # Stated explicitly rather than left to select_backend(): the whole
+ # execution pipeline is pinned to one substrate at its entry point.
+ backend=QueueBackend.PG,
fairness=FairnessKey(
org_id=context.organization_id,
workload_type=WorkloadType.NON_API,
diff --git a/workers/shared/models/execution_models.py b/workers/shared/models/execution_models.py
index ca5c33278..c24f7a534 100644
--- a/workers/shared/models/execution_models.py
+++ b/workers/shared/models/execution_models.py
@@ -10,10 +10,8 @@ from typing import TYPE_CHECKING, Any
# Import shared domain models from core
from unstract.core.data_models import (
- DEFAULT_WORKFLOW_TRANSPORT,
ExecutionStatus,
PreCreatedFileData,
- normalize_transport,
serialize_dataclass_to_dict,
)
@@ -390,11 +388,6 @@ class WorkflowContextData:
settings: dict[str, Any] | None = None
metadata: dict[str, Any] | None = None
is_scheduled: bool = False
- # Transport this execution rides end-to-end (9e). Populated from the inbound
- # task payload; the fan-out read that keeps the pipeline on one transport
- # lands in PR 2. Defaults to legacy Celery; coerced fail-closed in
- # __post_init__ so a garbage payload value can't reach the PR 2 read site.
- transport: str = DEFAULT_WORKFLOW_TRANSPORT
pre_created_file_executions: dict[str, PreCreatedFileData] = field(
default_factory=dict
)
@@ -424,16 +417,6 @@ class WorkflowContextData:
f"Must be one of: {valid_types}"
)
- # Coerce transport fail-closed: a garbage payload value (version skew,
- # bad PG JSONB) degrades to Celery here rather than reaching the PR 2
- # fan-out read. Unlike workflow_type (no safe default → raise), transport
- # is reversible by design, so it falls back instead of crashing.
- self.transport = normalize_transport(
- self.transport,
- logger=logger,
- context=f" (execution {self.execution_id})",
- )
-
@property
def file_count(self) -> int:
"""Get the number of files in this workflow context."""
diff --git a/workers/shared/models/file_processing.py b/workers/shared/models/file_processing.py
index b634d795e..af4062417 100644
--- a/workers/shared/models/file_processing.py
+++ b/workers/shared/models/file_processing.py
@@ -32,7 +32,6 @@ class FileProcessingContext:
workflow_logger: Any = None, # Type as Any to avoid import dependency
current_file_idx: int = 1,
total_files: int = 1,
- transport: str | None = None,
):
self.file_data = file_data
self.file_hash = file_hash
@@ -43,10 +42,6 @@ class FileProcessingContext:
self.workflow_logger = workflow_logger
self.current_file_idx = current_file_idx
self.total_files = total_files
- # Execution transport (celery | pg_queue). Authoritative here because
- # WorkerFileData carries no transport; threaded from the batch context so
- # the destination layer can apply PG-only guards.
- self.transport = transport
# Extract common identifiers
self.execution_id = file_data.execution_id
diff --git a/workers/shared/processing/files/processor.py b/workers/shared/processing/files/processor.py
index 52c4b9a11..90009d3a5 100644
--- a/workers/shared/processing/files/processor.py
+++ b/workers/shared/processing/files/processor.py
@@ -477,7 +477,6 @@ class FileProcessor:
workflow_file_execution_id: str = None,
workflow_file_execution_object: Any = None,
workflow_logger: WorkerWorkflowLogger = None,
- transport: str | None = None,
) -> FileProcessingResult:
"""Main orchestrator method that replaces the complex _process_file method.
@@ -513,7 +512,6 @@ class FileProcessor:
workflow_logger=workflow_logger,
current_file_idx=current_file_idx,
total_files=total_files,
- transport=transport,
)
logger.debug(
diff --git a/workers/shared/workflow/execution/orchestration_utils.py b/workers/shared/workflow/execution/orchestration_utils.py
index 80bfa0bc2..d40ac3663 100644
--- a/workers/shared/workflow/execution/orchestration_utils.py
+++ b/workers/shared/workflow/execution/orchestration_utils.py
@@ -9,14 +9,10 @@ from __future__ import annotations
import os
from typing import TYPE_CHECKING, Any
-from queue_backend import BarrierHandle, FairnessKey, get_barrier
+from queue_backend import BarrierHandle, FairnessKey
from queue_backend.pg_barrier import PgBarrier
-from unstract.core.data_models import (
- DEFAULT_WORKFLOW_TRANSPORT,
- ExecutionStatus,
- is_pg_transport,
-)
+from unstract.core.data_models import ExecutionStatus
from ...enums import FileDestinationType, PipelineType
from ...enums.worker_enums import QueueName
@@ -24,30 +20,9 @@ from ...infrastructure.logging import WorkerLogger
if TYPE_CHECKING:
from celery.canvas import Signature
- from queue_backend import Barrier
logger = WorkerLogger.get_logger(__name__)
-# Single ``Barrier`` instance reused across all
-# ``WorkflowOrchestrationUtils.create_chord_execution`` calls on the **celery**
-# transport. The substrate is selected at module-import (worker startup) time by
-# the ``WORKER_BARRIER_BACKEND`` env var (default ``chord``). Flag flips require a
-# pod restart — same posture as every other ``WORKER_*`` env in the codebase.
-_BARRIER = get_barrier()
-
-
-def _barrier_for_transport(transport: str) -> Barrier:
- """Pick the fan-in substrate for a per-execution transport (9e).
-
- The ``pg_queue`` transport always coordinates on Postgres — it uses a fresh
- :class:`PgBarrier` in its fire-and-forget mode regardless of
- ``WORKER_BARRIER_BACKEND`` (the env-selected singleton is the *celery*-transport
- substrate, which may be ``chord``). Every other transport uses that singleton.
- """
- if is_pg_transport(transport):
- return PgBarrier()
- return _BARRIER
-
class WorkflowOrchestrationUtils:
"""Centralized workflow orchestration patterns and utilities."""
@@ -62,13 +37,9 @@ class WorkflowOrchestrationUtils:
logger: Any,
workflow_logger: Any | None = None,
) -> None:
- """Record a **PG** orchestration failure: surface the error to the UI and
+ """Record an orchestration failure: surface the error to the UI and
reconcile the file counters, both best-effort.
- Call this ONLY on the PG transport (the caller gates on
- ``is_pg_transport``); the Celery failure path keeps its own original
- status update untouched, so it stays byte-identical.
-
Two things happen, neither of which may disturb the caller's control flow
(it still has the original orchestration exception to log / re-raise):
@@ -116,15 +87,14 @@ class WorkflowOrchestrationUtils:
app_instance: Any,
*,
fairness: FairnessKey | None = None,
- transport: str = DEFAULT_WORKFLOW_TRANSPORT,
) -> BarrierHandle | None:
- """Standardized fan-out + callback pattern (Phase 6 ``Barrier``).
+ """Standardized fan-out + callback pattern.
- Routes through ``CeleryChordBarrier`` — a thin wrapper around
- ``celery.chord(header)(body)`` that lets Phase 6b swap the
- substrate (e.g. to ``RedisDecrBarrier``) without touching this
- call site a second time. Behaviour is identical to the
- previous direct ``chord(...)`` call.
+ Routes through :class:`PgBarrier`, which arms a ``pg_barrier_state`` row
+ and dispatches the header tasks onto the PG queue. A fresh instance per
+ call: the barrier holds no cross-execution state, and the substrate is no
+ longer selectable, so there is nothing for a module-level singleton to
+ memoise.
Args:
batch_tasks: List of batch task signatures
@@ -139,22 +109,21 @@ class WorkflowOrchestrationUtils:
for the slot name constant.
Returns:
- Barrier handle (Celery ``AsyncResult``-shaped — ``.id``
- exposed for chord-id logging) or None if no batch tasks.
+ Barrier handle (``.id`` exposed for logging) or None if no
+ batch tasks.
Note:
CRITICAL: Returns None for zero batch tasks, signaling to
parent that direct pipeline status updates should be
handled instead.
"""
- return _barrier_for_transport(transport).enqueue(
+ return PgBarrier().enqueue(
batch_tasks,
callback_task_name=callback_task_name,
callback_kwargs=callback_kwargs,
callback_queue=callback_queue,
app_instance=app_instance,
fairness=fairness,
- transport=transport,
)
@staticmethod
diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py
index 0e076ebfc..a62253d51 100644
--- a/workers/shared/workflow/execution/service.py
+++ b/workers/shared/workflow/execution/service.py
@@ -25,7 +25,6 @@ from unstract.core.data_models import (
FileOperationConstants,
WorkflowDefinitionResponseData,
WorkflowEndpointConfigData,
- is_pg_transport,
)
# Import file execution tracking for proper recovery mechanism
@@ -1018,11 +1017,11 @@ class WorkerWorkflowExecutionService:
def _execute_structure_tool_workflow(
self, execution_service: WorkflowExecutionService, file_name: str
) -> None:
- """Execute structure tool as Celery task instead of Docker container.
+ """Execute structure tool as a task instead of a Docker container.
Calls execute_structure_tool directly (same process, in-band).
- Only the inner ExecutionDispatcher calls go through Celery to
- the executor worker.
+ Only the inner executor-RPC dispatches leave this process, and they go
+ to the executor worker over the PG queue.
"""
from file_processing.structure_tool_task import (
execute_structure_tool as _execute_structure_tool,
@@ -1362,36 +1361,30 @@ class WorkerWorkflowExecutionService:
logger.error("No connector_id found in any configuration source")
return None, {}
- def _pg_destination_already_written(
+ def _destination_already_written(
self,
*,
file_hash: FileHashData,
- transport: str | None,
use_file_history: bool,
workflow_id: str,
is_api: bool,
) -> bool:
"""True if this file's destination write already completed in a prior run.
- On the PG (at-least-once) transport a batch can be re-run after a crash
- or reaper-recovery, and such a re-run bypasses discovery's FileHistory
+ The queue is at-least-once, so a batch can be re-run after a crash or
+ reaper-recovery, and such a re-run bypasses discovery's FileHistory
filter — so re-check by hash (plus path for non-API) at the write
boundary. Returns True only if a COMPLETED FileHistory record already
exists for this file.
- Scoped two ways:
- - to the PG transport (the re-run duplicate this prevents is PG-specific;
- always False on Celery, so that path is unchanged), and
- - to ``use_file_history``: a workflow run with file history off is
- contractually "rewrite every run" (mirrors the discovery FileHistory
- filter), so its write is never skipped.
+ Scoped to ``use_file_history``: a workflow run with file history off is
+ contractually "rewrite every run" (mirrors the discovery FileHistory
+ filter), so its write is never skipped.
Fail-open on any lookup error (never block a legitimate write), logging
at error level so a persistent lookup regression is visible rather than a
silent, permanent no-op.
"""
- if not is_pg_transport(transport):
- return False
if not use_file_history:
return False
cache_key = file_hash.file_hash
@@ -1527,16 +1520,14 @@ class WorkerWorkflowExecutionService:
f"📤 File {file_hash.file_name} marked for DESTINATION processing - sending to {destination_display}"
)
- # PG-only re-run duplicate-write guard: a re-run after a crash /
+ # Re-run duplicate-write guard: a re-run after a crash /
# reaper-recovery bypasses discovery's FileHistory filter, so
# re-check by hash at the write boundary. If this file's
# destination write already completed in a prior run, skip it,
- # avoiding a duplicate destination write. A no-op on the Celery
- # transport, so that path is unchanged; fail-open so a lookup
+ # avoiding a duplicate destination write. Fail-open so a lookup
# error never blocks a write.
- if self._pg_destination_already_written(
+ if self._destination_already_written(
file_hash=file_hash,
- transport=file_processing_context.transport,
use_file_history=use_file_history,
workflow_id=workflow_id,
is_api=destination.is_api,
diff --git a/workers/tests/executor_dispatch_fakes.py b/workers/tests/executor_dispatch_fakes.py
new file mode 100644
index 000000000..09333f099
--- /dev/null
+++ b/workers/tests/executor_dispatch_fakes.py
@@ -0,0 +1,161 @@
+"""Shared fakes for executor request-reply dispatch tests.
+
+Several operation suites (summarize, table extract, agentic, …) assert the same
+two things about a tool dispatch: *which queue it lands on* and *what payload it
+carries*. They used to do that against the SDK's Celery ``ExecutionDispatcher``,
+mocking ``celery_app.send_task``. That dispatcher was deleted with the rest of
+the Celery transport (UN-4078); the surviving path is
+:class:`~unstract.workflow_execution.executor_rpc.PgExecutionDispatcher`, which
+enqueues through an injected :class:`QueueTransport` instead.
+
+This module is the single home for the fake transport those suites now share, so
+the retarget did not scatter ten near-identical mock classes across the tree.
+``test_executor_rpc.py`` keeps its own local ``_FakeTransport`` — it exercises
+failure modes (enqueue raising, poll raising, malformed rows) that the operation
+suites have no use for, and widening this one to cover them would couple the
+happy-path callers to that detail.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+# ``QUEUE_PREFIX`` is the queue-per-executor prefix. Named ``celery_executor_*``
+# for historical reasons and NOT dead Celery surface: ``worker-pg-executor``
+# subscribes to exactly these strings (chart ``workerPgExecutor``), so the names
+# are live wire contract. Imported from the production constant rather than
+# re-declared, so a rename cannot silently pass these tests while stranding the
+# real consumer.
+from unstract.workflow_execution.executor_rpc import (
+ QUEUE_PREFIX,
+ ExecResultRow,
+ queue_for_executor,
+)
+
+__all__ = [
+ "QUEUE_PREFIX",
+ "EagerExecutorTransport",
+ "FakeExecutorTransport",
+ "callback_signature",
+ "queue_for",
+]
+
+
+def queue_for(executor_name: str) -> str:
+ """The queue an executor's dispatches land on.
+
+ Delegates to the production helper rather than re-deriving the rule, so an
+ assertion here observes what the dispatcher actually does: a change to the
+ naming fails these suites instead of them agreeing with a stale copy.
+ """
+ return queue_for_executor(executor_name)
+
+
+def callback_signature(task_name: str, *, queue: str = "celery_callback") -> Any:
+ """A real ``Signature`` shaped the way PG self-chaining requires.
+
+ ``signature_to_continuation`` rejects a signature with no task name, no
+ queue, or any positional args. A bare ``MagicMock`` fails all three — every
+ attribute it auto-creates is truthy, so it trips the positional-args guard —
+ which makes it useless as a stand-in here. Imported by the suites that assert
+ on the continuation payload so they cannot drift on that shape.
+ """
+ from celery.canvas import Signature
+
+ return Signature(task_name, args=(), kwargs={}, queue=queue)
+
+
+class FakeExecutorTransport:
+ """Records ``enqueue`` calls and replays a canned result to the poll.
+
+ Exercises :class:`PgExecutionDispatcher` end-to-end with no database. The
+ recorded ``enqueue_calls`` are plain kwarg dicts — assert on
+ ``calls[0]["queue"]`` / ``["context"]`` / ``["org_id"]`` the way the old
+ suites asserted on ``send_task.call_args``.
+ """
+
+ def __init__(self, *, result: dict[str, Any] | None = None) -> None:
+ self.enqueue_calls: list[dict[str, Any]] = []
+ self.wait_timeouts: list[float] = []
+ self._result = result
+
+ def enqueue(
+ self,
+ *,
+ queue: str,
+ context: Any,
+ org_id: str,
+ reply_key: str | None = None,
+ on_success: Any | None = None,
+ on_error: Any | None = None,
+ task_id: str | None = None,
+ ) -> None:
+ # Mirrors the QueueTransport protocol exactly (keyword-only, fixed key
+ # set) rather than swallowing **kwargs: a production signature change
+ # then fails these suites instead of being silently absorbed.
+ self.enqueue_calls.append(
+ {
+ "queue": queue,
+ "context": context,
+ "org_id": org_id,
+ "reply_key": reply_key,
+ "on_success": on_success,
+ "on_error": on_error,
+ "task_id": task_id,
+ }
+ )
+
+ def wait_for_result(self, reply_key: str, timeout: float) -> ExecResultRow | None:
+ del reply_key
+ self.wait_timeouts.append(timeout)
+ if self._result is None:
+ return None
+ return ExecResultRow(status="completed", result=self._result, error="")
+
+ # -- convenience accessors, so callers don't index into the list inline ----
+
+ @property
+ def only_call(self) -> dict[str, Any]:
+ """The single recorded enqueue; fails loudly if there wasn't exactly one."""
+ assert len(self.enqueue_calls) == 1, (
+ f"expected exactly one enqueue, got {len(self.enqueue_calls)}"
+ )
+ return self.enqueue_calls[0]
+
+ @property
+ def queue(self) -> str:
+ """Queue of the single recorded enqueue."""
+ return self.only_call["queue"]
+
+ @property
+ def context(self) -> Any:
+ """``ExecutionContext`` of the single recorded enqueue."""
+ return self.only_call["context"]
+
+
+class EagerExecutorTransport(FakeExecutorTransport):
+ """Runs the ``execute_extraction`` task in-process instead of enqueueing it.
+
+ The full-chain contract test needs dispatcher → task → orchestrator → result to
+ actually execute. On the Celery transport that was done by patching
+ ``send_task`` to call ``task.apply()``; the equivalent here is a transport whose
+ ``enqueue`` runs the task synchronously and hands the outcome back through
+ ``wait_for_result``, so the same chain is exercised through the surviving
+ dispatcher.
+
+ ``args=[context.to_dict()]`` mirrors :class:`PgClientQueueTransport.enqueue`
+ exactly — if the real payload shape changes, this fake must change with it or
+ the chain it claims to cover is not the one that ships.
+ """
+
+ def __init__(self, task: Any) -> None:
+ super().__init__()
+ self._task = task
+
+ def enqueue(self, *, context: Any, **kwargs: Any) -> None:
+ # ``context`` is named so this fake breaks loudly if the protocol drops
+ # it; the rest is forwarded to the base recorder, which pins the full
+ # keyword-only key set.
+ super().enqueue(context=context, **kwargs)
+ outcome = self._task.apply(args=[context.to_dict()])
+ self._result = outcome.get()
diff --git a/workers/tests/test_agentic_operations.py b/workers/tests/test_agentic_operations.py
index 4d0fec497..713b5dc9d 100644
--- a/workers/tests/test_agentic_operations.py
+++ b/workers/tests/test_agentic_operations.py
@@ -8,10 +8,12 @@ from unittest.mock import MagicMock, patch
import pytest
from unstract.sdk1.execution.context import ExecutionContext, Operation
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.sdk1.execution.executor import BaseExecutor
from unstract.sdk1.execution.registry import ExecutorRegistry
from unstract.sdk1.execution.result import ExecutionResult
+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher
+
+from .executor_dispatch_fakes import FakeExecutorTransport, queue_for
AGENTIC_OPERATIONS = [
"agentic_extract",
@@ -108,19 +110,17 @@ class TestAgenticExecutorRegistration:
class TestAgenticQueueRouting:
def test_agentic_routes_to_correct_queue(self):
- queue = ExecutionDispatcher._get_queue("agentic")
+ queue = queue_for("agentic")
assert queue == "celery_executor_agentic"
@pytest.mark.parametrize("op", AGENTIC_OPERATIONS)
def test_dispatch_sends_to_agentic_queue(self, op):
- mock_app = MagicMock()
- mock_result = MagicMock()
- mock_result.get.return_value = ExecutionResult(
- success=True, data={"output": {}}
- ).to_dict()
- mock_app.send_task.return_value = mock_result
-
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
+ transport = FakeExecutorTransport(
+ result=ExecutionResult(
+ success=True, data={"output": {}}
+ ).to_dict()
+ )
+ dispatcher = PgExecutionDispatcher(transport)
ctx = ExecutionContext(
executor_name="agentic",
operation=op,
@@ -130,9 +130,7 @@ class TestAgenticQueueRouting:
)
dispatcher.dispatch(ctx)
- mock_app.send_task.assert_called_once()
- call_kwargs = mock_app.send_task.call_args
- assert call_kwargs.kwargs.get("queue") == "celery_executor_agentic"
+ assert transport.queue == "celery_executor_agentic"
# ---------------------------------------------------------------------------
diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.py
index a7c95ab5c..7bf65d182 100644
--- a/workers/tests/test_barrier.py
+++ b/workers/tests/test_barrier.py
@@ -1,29 +1,21 @@
-"""Characterisation tests for the ``Barrier`` abstraction (PG Queue Phase 6a).
-
-The two production chord call sites
-(``WorkflowOrchestrationUtils.create_chord_execution`` and
-``api-deployment/tasks.py``'s inline chord) were lifted behind
-``CeleryChordBarrier`` so a future Phase 6b can swap in
-``RedisDecrBarrier`` (or PG-based) implementations without touching
-call sites.
-
-Three layers of coverage:
-
-1. **Barrier protocol shape** — ``CeleryChordBarrier`` satisfies the
- ``Barrier`` Protocol and ``AsyncResult`` satisfies ``BarrierHandle``.
-2. **Wire equivalence with the pre-Barrier chord call** — given the
- same args, ``CeleryChordBarrier.enqueue(...)`` produces the same
- ``chord(header)(body)`` invocation, the same ``app.signature(...)``
- args, and the same return value. A regression that bypasses the
- abstraction or alters the wire surface fails one of these tests.
-3. **Fairness plumbing** — when ``fairness=`` is passed, the fairness
- header is attached to every enqueued task and to the callback
- (closes Phase 5.1's chord-fairness gap that was deliberately
- scoped out).
-
-The single ``chord(...)`` call still lives in
-``queue_backend/barrier.py`` (asserted by the inventory test in
-``test_chord_sites_characterisation.py``).
+"""Tests for the ``Barrier`` seam and its two production fan-out call sites.
+
+The seam exists because ``WorkflowOrchestrationUtils.create_chord_execution`` and
+``api-deployment/tasks.py`` both fan out and both need one dispatch point. It
+once carried three implementations selected by ``WORKER_BARRIER_BACKEND``;
+UN-4078 left only :class:`PgBarrier`.
+
+Two layers of coverage:
+
+1. **Protocol shape** — the surviving implementation satisfies ``Barrier``, and
+ the handle satisfies ``BarrierHandle``.
+2. **Call-site contracts** — the api-deployment fan-out passes API fairness,
+ handles the zero-batch branch, raises rather than silently falling back on a
+ falsy result with non-empty batches, and routes manual review per batch.
+
+The wire-equivalence layer (chord call shape, AMQP fairness headers) went with
+``CeleryChordBarrier``; the equivalent PG assertions live in
+``test_pg_barrier.py``.
"""
from __future__ import annotations
@@ -36,17 +28,17 @@ import pytest
from queue_backend import (
Barrier,
BarrierHandle,
- CeleryChordBarrier,
FairnessKey,
)
-from queue_backend.fairness import FAIRNESS_HEADER_NAME, WorkloadType
+from queue_backend.fairness import WorkloadType
@pytest.fixture
def app() -> MagicMock:
"""Celery-app-shaped mock with a working ``.signature(...)``.
- Used by every test that drives ``CeleryChordBarrier.enqueue(...)``.
+ The fan-out call sites still build header signatures through a Celery app
+ before handing them to the barrier, so they still need one.
Shared via fixture rather than a per-class ``_make_app`` helper to
keep test classes free of boilerplate (SonarCloud S4144
duplication).
@@ -74,13 +66,16 @@ def mock_chord():
class TestBarrierProtocolShape:
- def test_celery_chord_barrier_satisfies_protocol(self):
- """``CeleryChordBarrier`` is structurally a ``Barrier``.
+ def test_pg_barrier_satisfies_protocol(self):
+ """The one remaining implementation is structurally a ``Barrier``.
- Pinned so Phase 6b's ``RedisDecrBarrier`` can be substituted at
- any call site without runtime ``AttributeError``.
+ Kept after ``CeleryChordBarrier`` was deleted: the Protocol still exists
+ because two call sites program against it, so something must assert the
+ implementation actually satisfies it.
"""
- barrier: Barrier = CeleryChordBarrier()
+ from queue_backend import PgBarrier
+
+ barrier: Barrier = PgBarrier()
assert callable(getattr(barrier, "enqueue", None))
def test_barrier_handle_protocol_satisfied_by_celery_async_result(self):
@@ -151,247 +146,30 @@ class TestBarrierProtocolShape:
# --- Wire equivalence with the pre-Barrier chord call ---
-class TestCeleryChordBarrierWireEquivalence:
- """``CeleryChordBarrier.enqueue(...)`` must produce the same
- ``chord(batch_tasks)(callback_signature)`` invocation that the
- original inline ``chord(...)`` calls produced — modulo the
- additive ``x-fairness-key`` header that the barrier optionally
- stamps onto signatures.
- """
-
- def test_empty_header_returns_none_and_skips_chord(self, app, mock_chord):
- """Zero-batch contract preserved: barrier returns None,
- ``chord(...)`` is never called.
-
- Parent callers (``general/tasks.py``, ``api-deployment/tasks.py``)
- rely on this signal to handle pipeline status updates directly.
- """
- result = CeleryChordBarrier().enqueue(
- [],
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-1", "pipeline_id": "pipe-1"},
- callback_queue="file_processing_callback",
- app_instance=app,
- )
-
- assert result is None
- mock_chord.assert_not_called()
-
- def test_non_empty_header_invokes_chord_header_then_body(self, app, mock_chord):
- """The original ``chord(header)(body)`` two-step call must
- be preserved exactly — a refactor that flattens or reorders
- these calls would change Celery's chord semantics."""
- header = [MagicMock(name="h1"), MagicMock(name="h2")]
-
- CeleryChordBarrier().enqueue(
- header,
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="file_processing_callback",
- app_instance=app,
- )
-
- # Step 1: chord(header)
- mock_chord.assert_called_once_with(header)
- # Step 2: chord_obj(callback_signature) — applies the chord
- mock_chord.return_value.assert_called_once_with(app.signature.return_value)
-
- def test_callback_signature_args_match_pre_uplift_contract(self, app, mock_chord):
- """``app.signature(task_name, kwargs=..., queue=...)`` is
- called with exactly the args the pre-Barrier helper used."""
- callback_kwargs = {
- "execution_id": "exec-42",
- "pipeline_id": "pipe-7",
- "organization_id": "org-x",
- }
-
- CeleryChordBarrier().enqueue(
- [MagicMock(name="h")],
- callback_task_name="process_batch_callback_api",
- callback_kwargs=callback_kwargs,
- callback_queue="api_file_processing_callback",
- app_instance=app,
- )
-
- # Without ``fairness=``, no ``headers=`` kwarg on the signature
- # call — matches the pre-Barrier wire byte-for-byte.
- app.signature.assert_called_once_with(
- "process_batch_callback_api",
- kwargs=callback_kwargs,
- queue="api_file_processing_callback",
- )
-
- def test_returns_chord_result_object(self, app, mock_chord):
- """The barrier handle is whatever ``chord(header)(body)``
- returns — Celery's ``AsyncResult`` in production. Callers
- log ``.id`` for chord-id tracing."""
- chord_result = MagicMock(name="chord_result")
- mock_chord.return_value.return_value = chord_result
-
- result = CeleryChordBarrier().enqueue(
- [MagicMock()],
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="file_processing_callback",
- app_instance=app,
- )
-
- assert result is chord_result
-
- def test_chord_failure_is_re_raised_after_logging(self, app, mock_chord):
- """If ``chord(...)`` raises (broker outage, serialisation
- error, etc.), the barrier logs and re-raises — never swallows.
-
- Without this, callers would treat broker failures as silent
- zero-task chords and skip pipeline status updates entirely.
- """
- mock_chord.side_effect = RuntimeError("broker exploded")
-
- with pytest.raises(RuntimeError, match="broker exploded"):
- CeleryChordBarrier().enqueue(
- [MagicMock()],
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="file_processing_callback",
- app_instance=app,
- )
-
-
-# --- Fairness plumbing ---
-
-
-class TestCeleryChordBarrierFairnessHeader:
- """When ``fairness=FairnessKey(...)`` is passed, the barrier
- stamps ``x-fairness-key`` on every header task and the callback.
-
- Closes the chord-fairness gap from Phase 5.1: bare ``dispatch()``
- calls carried fairness; the two chord call sites didn't (they
- bypassed ``dispatch()`` entirely). After this Phase 6a uplift,
- every queue-crossing payload on the workflow-execution path now
- carries the fairness slot.
- """
-
- def test_fairness_header_stamped_on_callback_signature(self, app, mock_chord):
- CeleryChordBarrier().enqueue(
- [MagicMock(name="h")],
- callback_task_name="process_batch_callback_api",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="api_file_processing_callback",
- app_instance=app,
- fairness=FairnessKey(org_id="org-1", workload_type=WorkloadType.API),
- )
-
- headers = app.signature.call_args.kwargs.get("headers")
- assert headers is not None, (
- "expected ``headers=`` kwarg on app.signature when fairness "
- "is passed"
- )
- assert headers[FAIRNESS_HEADER_NAME] == {
- "org_id": "org-1",
- "workload_type": "api",
- "pipeline_priority": 5,
- }
-
- def test_fairness_header_stamped_on_every_header_task(self, app, mock_chord):
- """Every batch task in the header carries the fairness header
- so PG Queue's per-task fairness scheduler can route each
- independently.
-
- Header signatures are stamped via ``Signature.clone().set(...)``
- (not in-place ``.set(...)``) — clone avoids cross-tenant
- header leakage if a future retry path or signature cache ever
- re-uses the original ``header_tasks`` list with a different
- ``FairnessKey``. The test asserts ``.clone().set(headers=...)``
- is called on each original task.
- """
- h1, h2, h3 = (MagicMock(name=f"header_task_{i}") for i in range(3))
-
- CeleryChordBarrier().enqueue(
- [h1, h2, h3],
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="file_processing_callback",
- app_instance=app,
- fairness=FairnessKey(org_id="org-1", workload_type=WorkloadType.NON_API),
- )
-
- expected = {
- FAIRNESS_HEADER_NAME: {
- "org_id": "org-1",
- "workload_type": "non_api",
- "pipeline_priority": 5,
- }
- }
- for task in (h1, h2, h3):
- # ``.clone()`` produces a fresh signature; the ``.set(...)``
- # then attaches the fairness header to the clone, leaving
- # the original ``task`` unchanged.
- task.clone.assert_called_once_with()
- task.clone.return_value.set.assert_called_once_with(headers=expected)
- # Direct ``.set(...)`` on the original is NEVER called —
- # the whole point of the clone-and-set pattern.
- task.set.assert_not_called()
-
- def test_no_fairness_no_header_added(self, app, mock_chord):
- """When ``fairness=None``, the barrier behaves byte-for-byte
- like the pre-Barrier chord call — no ``headers=`` kwarg on
- the signature, no ``.set(headers=...)`` on header tasks.
-
- Mixed-version rolling deploys are safe because this branch
- produces the same wire as the old direct chord call.
- """
- header = [MagicMock(name="h1"), MagicMock(name="h2")]
-
- CeleryChordBarrier().enqueue(
- header,
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="file_processing_callback",
- app_instance=app,
- )
+class TestCreateChordExecutionUsesPgBarrier:
+ """Pin ``WorkflowOrchestrationUtils.create_chord_execution`` to :class:`PgBarrier`.
- # Callback signature: no headers kwarg.
- assert "headers" not in app.signature.call_args.kwargs
-
- # Header tasks: ``.set(...)`` not called for headers stamping.
- for task in header:
- # ``set`` might be called for other reasons in production,
- # but never with ``headers=`` when fairness is None.
- for call in task.set.call_args_list:
- assert "headers" not in call.kwargs, (
- "unexpected fairness header stamped without fairness="
- )
-
-
-# --- Singleton routing (pins the _BARRIER dispatch point) ---
-
-
-class TestOrchestrationUtilsRoutesThroughSingleton:
- """Pin the ``WorkflowOrchestrationUtils.create_chord_execution``
- routing to the module-level ``_BARRIER`` singleton.
-
- A refactor that bypasses the singleton (e.g. inlining
- ``CeleryChordBarrier().enqueue(...)`` inside the static method,
- or going back to a direct ``chord(...)`` call) would still pass
- the inventory canary in ``test_chord_sites_characterisation.py``
- as long as ``chord(...)`` lives somewhere inside ``barrier.py``.
- This test closes that gap so the singleton is the unambiguous
- single dispatch point — ready for Phase 6b's factory swap.
+ Replaces the old singleton-routing suite. There used to be a module-level
+ ``_BARRIER`` chosen by ``WORKER_BARRIER_BACKEND`` plus a per-execution
+ ``transport`` that decided between it and a fresh ``PgBarrier``; both went
+ with the Celery transport (UN-4078). What is worth pinning now is that the
+ call site still funnels through one substrate and forwards its arguments
+ unchanged — a refactor that reached for a barrier directly, or dropped
+ ``fairness`` on the way through, would otherwise pass silently.
"""
- def test_create_chord_execution_delegates_to_module_barrier(self):
+ def test_delegates_to_pg_barrier_with_arguments_intact(self):
from shared.workflow.execution import orchestration_utils
from shared.workflow.execution.orchestration_utils import (
WorkflowOrchestrationUtils,
)
app_mock = MagicMock(name="celery_app")
- app_mock.signature.return_value = MagicMock(name="callback_signature")
batch = [MagicMock(name="h1")]
fairness = FairnessKey(org_id="org-1", workload_type=WorkloadType.API)
- with patch.object(orchestration_utils, "_BARRIER") as mock_barrier:
- mock_barrier.enqueue.return_value = MagicMock(name="result")
+ with patch.object(orchestration_utils, "PgBarrier") as barrier_cls:
+ barrier_cls.return_value.enqueue.return_value = MagicMock(name="result")
WorkflowOrchestrationUtils.create_chord_execution(
batch_tasks=batch,
callback_task_name="process_batch_callback_api",
@@ -401,54 +179,36 @@ class TestOrchestrationUtilsRoutesThroughSingleton:
fairness=fairness,
)
- # Default transport (celery) → the env-selected singleton, with the
- # per-execution transport threaded through to the substrate (9e).
- mock_barrier.enqueue.assert_called_once_with(
+ barrier_cls.return_value.enqueue.assert_called_once_with(
batch,
callback_task_name="process_batch_callback_api",
callback_kwargs={"execution_id": "exec-1"},
callback_queue="api_file_processing_callback",
app_instance=app_mock,
fairness=fairness,
- transport="celery",
)
- def test_pg_queue_transport_bypasses_singleton_for_pgbarrier(self):
- # 9e: a pg_queue execution must coordinate on Postgres regardless of
- # WORKER_BARRIER_BACKEND, so it uses a fresh PgBarrier — NOT the (possibly
- # chord) singleton. Pin that the singleton's enqueue is never called.
- from shared.workflow.execution import orchestration_utils
+ def test_no_transport_argument_is_accepted(self):
+ """The per-execution ``transport`` kwarg is gone, not merely ignored.
+
+ It was the field that let a stale payload select a Celery fan-out after
+ the Celery consumers were scaled to zero. Accepting and discarding it
+ would leave that call shape looking valid.
+ """
from shared.workflow.execution.orchestration_utils import (
WorkflowOrchestrationUtils,
)
- app_mock = MagicMock(name="celery_app")
- batch = [MagicMock(name="h1")]
-
- with (
- patch.object(orchestration_utils, "_BARRIER") as mock_singleton,
- patch.object(orchestration_utils, "PgBarrier") as mock_pgbarrier_cls,
- ):
+ with pytest.raises(TypeError):
WorkflowOrchestrationUtils.create_chord_execution(
- batch_tasks=batch,
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-pg"},
- callback_queue="general",
- app_instance=app_mock,
- transport="pg_queue",
+ batch_tasks=[MagicMock()],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ transport="celery",
)
- mock_singleton.enqueue.assert_not_called() # NOT the env singleton
- mock_pgbarrier_cls.assert_called_once_with() # a fresh PgBarrier
- mock_pgbarrier_cls.return_value.enqueue.assert_called_once()
- assert (
- mock_pgbarrier_cls.return_value.enqueue.call_args.kwargs["transport"]
- == "pg_queue"
- )
-
-
-# --- Executing tests for call-site fairness contracts ---
-
def _load_api_deployment_tasks():
"""Load ``api-deployment/tasks.py`` as a module.
diff --git a/workers/tests/test_barrier_backend_selection.py b/workers/tests/test_barrier_backend_selection.py
deleted file mode 100644
index 0948814e6..000000000
--- a/workers/tests/test_barrier_backend_selection.py
+++ /dev/null
@@ -1,123 +0,0 @@
-"""Tests for the ``get_barrier()`` factory + ``WORKER_BARRIER_BACKEND`` flag.
-
-Phase 6b selects the ``Barrier`` substrate at runtime via an env flag.
-This test suite pins:
-
-- Default (env unset) → ``CeleryChordBarrier`` — protects against any
- production behaviour change at merge time; downstream envs that
- haven't added the new flag still run the same substrate they ran
- yesterday.
-- ``WORKER_BARRIER_BACKEND=chord`` → ``CeleryChordBarrier``.
-- ``WORKER_BARRIER_BACKEND=redis`` → ``RedisDecrBarrier``.
-- Unknown values raise ``ValueError`` — silent fallback to default
- would mask a typo'd flag in production env, which the canary
- must not allow.
-- Case + whitespace tolerance — operators frequently set env vars
- via shell scripts or k8s configmaps; ``" REDIS "`` should
- resolve the same as ``"redis"``.
-- ``BarrierBackend`` enum membership — the canonical string values
- match what operators set in env.
-"""
-
-from __future__ import annotations
-
-import pytest
-
-from queue_backend import (
- BarrierBackend,
- CeleryChordBarrier,
- PgBarrier,
- RedisDecrBarrier,
- get_barrier,
-)
-
-
-class TestBarrierBackendEnum:
- def test_chord_value_is_canonical_string(self):
- """The enum value is what operators set in env. If this
- drifts, the factory's dispatch breaks silently for one
- substrate (the one whose enum value no longer matches the
- documented env value)."""
- assert BarrierBackend.CHORD.value == "chord"
- assert BarrierBackend.REDIS.value == "redis"
- assert BarrierBackend.PG.value == "pg"
-
- def test_str_enum_string_equality(self):
- """``StrEnum`` members compare equal to their string values —
- means tests + production code can use either form
- interchangeably without conversion."""
- assert BarrierBackend.CHORD == "chord"
- assert BarrierBackend.REDIS == "redis"
- assert BarrierBackend.PG == "pg"
-
-
-class TestGetBarrierFactory:
- def test_default_returns_celery_chord_barrier(self, monkeypatch):
- """Env unset → ``CeleryChordBarrier``. This is what protects
- production from a behaviour change at merge time — downstream
- envs (``unstract-cloud``, self-hosted, etc.) that haven't
- added the new flag run the same substrate they ran yesterday."""
- monkeypatch.delenv("WORKER_BARRIER_BACKEND", raising=False)
- barrier = get_barrier()
- assert isinstance(barrier, CeleryChordBarrier)
-
- def test_chord_selects_celery_chord_barrier(self, monkeypatch):
- monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.CHORD)
- assert isinstance(get_barrier(), CeleryChordBarrier)
-
- def test_redis_selects_redis_decr_barrier(self, monkeypatch):
- monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.REDIS)
- assert isinstance(get_barrier(), RedisDecrBarrier)
-
- def test_pg_selects_pg_barrier(self, monkeypatch):
- monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.PG)
- assert isinstance(get_barrier(), PgBarrier)
-
- def test_unknown_value_raises_loudly(self, monkeypatch):
- """Silent fallback to default would mask a typo'd production
- env (``rediz``, ``REDDIS``, ``redis-decr``) — the canary
- must surface the misconfiguration."""
- monkeypatch.setenv("WORKER_BARRIER_BACKEND", "rediz")
- with pytest.raises(ValueError, match=r"rediz"):
- get_barrier()
-
- def test_empty_string_raises_loudly(self, monkeypatch):
- """Empty string is not the same as "unset" — it's a positive
- misconfiguration (e.g. an unsubstituted ``$BARRIER`` in a
- shell script). Surface it rather than falling back to
- default, which would mask the bug."""
- monkeypatch.setenv("WORKER_BARRIER_BACKEND", "")
- with pytest.raises(ValueError):
- get_barrier()
-
- def test_case_insensitive(self, monkeypatch):
- """Operators frequently set env vars via shell heredocs or
- k8s configmaps with inconsistent casing. ``REDIS`` and
- ``Redis`` should resolve the same as ``redis``."""
- monkeypatch.setenv("WORKER_BARRIER_BACKEND", "REDIS")
- assert isinstance(get_barrier(), RedisDecrBarrier)
- monkeypatch.setenv("WORKER_BARRIER_BACKEND", "Chord")
- assert isinstance(get_barrier(), CeleryChordBarrier)
-
- def test_whitespace_tolerated(self, monkeypatch):
- """Stray whitespace (common from shell substitutions) is
- stripped before the membership check."""
- monkeypatch.setenv("WORKER_BARRIER_BACKEND", " redis ")
- assert isinstance(get_barrier(), RedisDecrBarrier)
-
- def test_returns_fresh_instance_per_call(self, monkeypatch):
- """``get_barrier()`` builds a new instance each call rather
- than returning a singleton. Call sites that want a singleton
- (e.g. ``orchestration_utils._BARRIER``) cache it at module
- load time. Tests can flip the env per-test without leaking
- a stale instance from a previous test."""
- monkeypatch.setenv("WORKER_BARRIER_BACKEND", "chord")
- a = get_barrier()
- b = get_barrier()
- assert a is not b
- assert isinstance(a, CeleryChordBarrier)
- assert isinstance(b, CeleryChordBarrier)
-
-
-if __name__ == "__main__":
- pytest.main([__file__, "-v"])
diff --git a/workers/tests/test_barrier_differential.py b/workers/tests/test_barrier_differential.py
deleted file mode 100644
index 99bbb1200..000000000
--- a/workers/tests/test_barrier_differential.py
+++ /dev/null
@@ -1,232 +0,0 @@
-"""Differential test — both ``Barrier`` substrates produce equivalent
-callback firings for a fixed input.
-
-This is the load-bearing test that justifies the substrate swap:
-given the same header tasks + callback descriptor + fairness, both
-``CeleryChordBarrier`` and ``RedisDecrBarrier`` must ultimately fire
-the same callback with the same ``(task_name, args, kwargs, queue,
-headers)``. The execution paths diverge — chord aggregates via
-Celery's chord backend, redis aggregates via ``DECR remaining`` +
-``RPUSH`` — but the observable output (what the callback receives)
-is equivalent.
-
-If this test fails, the substrate swap is *not* safe to roll out:
-the callback would see different aggregated results or different
-metadata between substrates, breaking executions when the flag
-flips.
-
-Note this is a unit-level differential — it pins the wire shape of
-the callback dispatch, not full end-to-end equivalence. A future
-integration test (against real Celery + Redis with N parallel
-workers) would belong in ``test_barrier_integration.py``.
-"""
-
-from __future__ import annotations
-
-import json
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from queue_backend import CeleryChordBarrier, RedisDecrBarrier, barrier_decr_and_check
-from queue_backend.fairness import FairnessKey, WorkloadType
-
-
-# A fixed scenario both substrates run through.
-_HEADER_TASK_RESULTS = [
- {"batch_id": "b1", "files": ["f1", "f2"], "status": "ok"},
- {"batch_id": "b2", "files": ["f3", "f4"], "status": "ok"},
- {"batch_id": "b3", "files": ["f5"], "status": "ok"},
-]
-_CALLBACK_TASK = "process_batch_callback_api"
-_CALLBACK_KWARGS = {
- "execution_id": "exec-differential-1",
- "pipeline_id": "pipe-x",
- "organization_id": "org-y",
-}
-_CALLBACK_QUEUE = "api_file_processing_callback"
-_FAIRNESS = FairnessKey(org_id="org-y", workload_type=WorkloadType.API)
-
-
-def _capture_chord_callback_dispatch() -> dict:
- """Run ``CeleryChordBarrier.enqueue`` against a mocked ``chord``
- and return the callback signature's construction args.
-
- Chord's wire model: ``chord(header_tasks)(callback_signature)`` —
- the callback_signature is built via ``app.signature(name, kwargs=...,
- queue=..., headers=...)`` and then passed as the body.
- """
- app = MagicMock(name="app")
- headers_captured = {}
-
- def signature_recorder(name, **kwargs):
- headers_captured["task_name"] = name
- headers_captured["args"] = kwargs.get("args") # None for chord callback
- headers_captured["kwargs"] = kwargs.get("kwargs")
- headers_captured["queue"] = kwargs.get("queue")
- headers_captured["headers"] = kwargs.get("headers")
- return MagicMock(name="callback_signature")
-
- app.signature.side_effect = signature_recorder
- header = [MagicMock(name=f"h{i}") for i in range(len(_HEADER_TASK_RESULTS))]
-
- with patch("queue_backend.barrier.chord"):
- CeleryChordBarrier().enqueue(
- header,
- callback_task_name=_CALLBACK_TASK,
- callback_kwargs=_CALLBACK_KWARGS,
- callback_queue=_CALLBACK_QUEUE,
- app_instance=app,
- fairness=_FAIRNESS,
- )
- return headers_captured
-
-
-def _capture_redis_callback_dispatch() -> dict:
- """Drive ``RedisDecrBarrier.enqueue`` + simulate the link task
- completing for all header tasks, then capture the final
- callback signature's construction args at the post-aggregation
- dispatch site.
- """
- redis_mock = MagicMock(name="redis")
- barrier = RedisDecrBarrier(redis_client=redis_mock)
- header = [MagicMock(name=f"h{i}") for i in range(len(_HEADER_TASK_RESULTS))]
- barrier.enqueue(
- header,
- callback_task_name=_CALLBACK_TASK,
- callback_kwargs=_CALLBACK_KWARGS,
- callback_queue=_CALLBACK_QUEUE,
- app_instance=MagicMock(name="app"),
- fairness=_FAIRNESS,
- )
-
- # Now simulate each link firing. The first N-1 see remaining > 0;
- # the last sees remaining == 0 and dispatches the callback.
- headers_captured = {}
-
- def signature_recorder(name, **kwargs):
- headers_captured["task_name"] = name
- headers_captured["args"] = kwargs.get("args")
- headers_captured["kwargs"] = kwargs.get("kwargs")
- headers_captured["queue"] = kwargs.get("queue")
- headers_captured["headers"] = kwargs.get("headers")
- cb_sig = MagicMock(name="callback_signature")
- cb_sig.apply_async.return_value = MagicMock(id="callback-task-id")
- return cb_sig
-
- # Simulate the Lua script's return for each link invocation.
- # First N-1 calls: [remaining, []]. Last call: [0, [serialised results]].
- n = len(_HEADER_TASK_RESULTS)
- serialised_results = [json.dumps(r) for r in _HEADER_TASK_RESULTS]
- return_values = [
- [n - 1 - i, []] for i in range(n - 1)
- ] + [[0, serialised_results]]
- script_mock = MagicMock(side_effect=return_values)
-
- with patch("queue_backend.redis_barrier._get_redis_client") as get_redis:
- # Inside the link task, ``_get_redis_client()`` is called to
- # build the script invocation. Return a mock whose
- # register_script returns the script_mock that yields the
- # sequenced return values.
- link_redis = MagicMock(name="link_redis")
- link_redis.register_script.return_value = script_mock
- get_redis.return_value = link_redis
-
- with patch("celery.current_app") as current_app:
- current_app.signature.side_effect = signature_recorder
- for header_result in _HEADER_TASK_RESULTS:
- barrier_decr_and_check.run(
- header_result,
- execution_id=_CALLBACK_KWARGS["execution_id"],
- callback_descriptor={
- "task_name": _CALLBACK_TASK,
- "kwargs": _CALLBACK_KWARGS,
- "queue": _CALLBACK_QUEUE,
- "fairness_headers": _FAIRNESS.as_header(),
- },
- )
- return headers_captured
-
-
-class TestBarrierSubstrateDifferential:
- """Both substrates must produce equivalent callback firings.
-
- A failure here means the swap is not safe — flag flipping in
- Phase 6c would change the data the callback receives.
- """
-
- def test_same_callback_task_name(self):
- """Both substrates fire the same callback task name."""
- chord_cap = _capture_chord_callback_dispatch()
- redis_cap = _capture_redis_callback_dispatch()
- assert chord_cap["task_name"] == redis_cap["task_name"] == _CALLBACK_TASK
-
- def test_same_callback_kwargs(self):
- """Both substrates forward the same ``execution_id`` /
- ``pipeline_id`` / ``organization_id`` to the callback."""
- chord_cap = _capture_chord_callback_dispatch()
- redis_cap = _capture_redis_callback_dispatch()
- assert chord_cap["kwargs"] == redis_cap["kwargs"] == _CALLBACK_KWARGS
-
- def test_same_callback_queue(self):
- """Both substrates route the callback to the same queue —
- without this, post-Phase-6c the callback could land on a
- worker that doesn't have it registered."""
- chord_cap = _capture_chord_callback_dispatch()
- redis_cap = _capture_redis_callback_dispatch()
- assert chord_cap["queue"] == redis_cap["queue"] == _CALLBACK_QUEUE
-
- def test_same_fairness_headers(self):
- """Both substrates ride the same ``x-fairness-key`` AMQP
- header on the callback. Mismatch would mean post-flag-flip
- executions route through different fairness slots, which
- the L1/L2/L3 admission logic could throttle differently."""
- chord_cap = _capture_chord_callback_dispatch()
- redis_cap = _capture_redis_callback_dispatch()
- assert chord_cap["headers"] == redis_cap["headers"] == _FAIRNESS.as_header()
-
- def test_redis_substrate_passes_aggregated_results_as_first_arg(self):
- """Chord's callback receives ``[result_1, result_2, ...]`` via
- Celery's chord-aggregator (an implicit first positional arg
- Celery injects). The Redis substrate must replicate the
- same shape — pass the aggregated list explicitly via
- ``args=[all_results]`` on the callback signature.
-
- We can't observe chord's implicit arg injection from the
- outside (it's the chord backend's job), but we CAN pin the
- Redis substrate's behaviour: when aggregation completes,
- the callback signature's ``args[0]`` MUST equal the
- original header-task results list. That's the contract the
- callback already depends on (``file_batch_results: list[dict]``
- — see callback/tasks.py:1552)."""
- redis_cap = _capture_redis_callback_dispatch()
- assert redis_cap["args"] == [_HEADER_TASK_RESULTS], (
- f"Redis substrate's callback args must be a list "
- f"containing one list-of-results (matching chord's "
- f"aggregator shape). Got: {redis_cap['args']!r}"
- )
-
- def test_both_handle_empty_headers_identically(self):
- """Empty header → ``None`` from both substrates. The caller's
- ``if result is None:`` branch then handles the zero-files
- contract uniformly."""
- chord_result = CeleryChordBarrier().enqueue(
- [],
- callback_task_name=_CALLBACK_TASK,
- callback_kwargs=_CALLBACK_KWARGS,
- callback_queue=_CALLBACK_QUEUE,
- app_instance=MagicMock(),
- )
- redis_result = RedisDecrBarrier(redis_client=MagicMock()).enqueue(
- [],
- callback_task_name=_CALLBACK_TASK,
- callback_kwargs=_CALLBACK_KWARGS,
- callback_queue=_CALLBACK_QUEUE,
- app_instance=MagicMock(),
- )
- assert chord_result is None
- assert redis_result is None
-
-
-if __name__ == "__main__":
- pytest.main([__file__, "-v"])
diff --git a/workers/tests/test_chord_callback_boundary.py b/workers/tests/test_chord_callback_boundary.py
index 055ff69cc..dc5114e8a 100644
--- a/workers/tests/test_chord_callback_boundary.py
+++ b/workers/tests/test_chord_callback_boundary.py
@@ -21,6 +21,7 @@ from __future__ import annotations
import json
from types import SimpleNamespace
+from unittest import mock
from unittest.mock import MagicMock
import pytest
@@ -767,13 +768,17 @@ class TestRealConsumerTolerance:
assert aggregated["batches_processed"] == 2
-class TestProcessFileBatchTransportRouting:
- """9e PR 2c: ``_process_file_batch_core`` routes by ``barrier_context`` —
- Celery chord path runs the stages directly (the chord ``.link`` decrements);
- the PG path wraps them in ``run_batch_with_barrier`` (in-body claim + decrement).
+class TestProcessFileBatchBarrierRouting:
+ """``_process_file_batch_core`` routes by ``barrier_context``.
+
+ With one present (every real fan-out injects it) the stages are wrapped in
+ ``run_batch_with_barrier`` for the in-body claim + decrement. With it absent
+ — now only reachable via a malformed payload, since the Celery ``.link``
+ path that legitimately produced ``None`` is gone — the stages still run, but
+ nothing decrements a barrier, so the call is expected to log loudly.
"""
- def test_celery_path_runs_stages_directly(self, monkeypatch):
+ def test_missing_barrier_context_runs_stages_and_logs(self, monkeypatch):
from file_processing import tasks as tasks_mod
stages_kwargs = {}
@@ -789,12 +794,15 @@ class TestProcessFileBatchTransportRouting:
"run_batch_with_barrier",
lambda *a, **k: barrier_calls.append(1),
)
- out = tasks_mod._process_file_batch_core(MagicMock(), {"fb": 1}, None)
+ with mock.patch.object(tasks_mod.logger, "error") as log_error:
+ out = tasks_mod._process_file_batch_core(MagicMock(), {"fb": 1}, None)
assert out == {"r": "celery"}
- assert barrier_calls == [] # barrier path NOT taken on the celery transport
- assert stages_kwargs.get("is_pg") is False # UN-3662 guard OFF on Celery
+ assert barrier_calls == [] # no barrier to wrap
+ # The batch still runs, but the operator must be told the callback will
+ # never fire — a silent run here would strand the execution to the reaper.
+ log_error.assert_called_once()
- def test_pg_path_routes_through_barrier(self, monkeypatch):
+ def test_barrier_context_routes_through_barrier(self, monkeypatch):
from file_processing import tasks as tasks_mod
stages_kwargs = {}
@@ -818,7 +826,6 @@ class TestProcessFileBatchTransportRouting:
assert captured["ctx"] is ctx
assert out["via"] == "barrier"
assert out["work"] == {"r": "work"} # work_fn runs the real stages
- assert stages_kwargs.get("is_pg") is True # UN-3662 guard ON on PG
if __name__ == "__main__":
diff --git a/workers/tests/test_chord_sites_characterisation.py b/workers/tests/test_chord_sites_characterisation.py
index 5732ef44e..1b683140c 100644
--- a/workers/tests/test_chord_sites_characterisation.py
+++ b/workers/tests/test_chord_sites_characterisation.py
@@ -1,365 +1,110 @@
-"""Characterisation tests for the chord-callback boundary, now lifted
-behind the ``Barrier`` abstraction (PG Queue Phase 6a).
-
-Originally written to lock down the inline ``celery.chord(...)`` calls
-at two call sites before they were lifted. Post-uplift the inline
-calls are gone — both call sites delegate to
-``WorkflowOrchestrationUtils.create_chord_execution`` which in turn
-delegates to ``CeleryChordBarrier.enqueue(...)``. The single remaining
-``chord(...)`` call lives inside ``workers/queue_backend/barrier.py``.
-
-Sites covered (assertions unchanged from the original pre-uplift
-suite — they characterise the **same** ``chord(header)(body)``
-contract, now reached one indirection later):
-1. ``shared/workflow/execution/orchestration_utils.py`` —
- ``WorkflowOrchestrationUtils.create_chord_execution`` (centralised helper).
-2. ``api-deployment/tasks.py`` — same helper, called inline inside
- ``_run_workflow_api`` (post-uplift it no longer calls ``chord(...)``
- directly; covered by the inventory test below).
-
-Chord is the highest-risk Celery construct called out in the PG Queue
-decision doc (silent task drops at ~130K-task scale). A future
-Phase 6b will introduce ``RedisDecrBarrier`` as a second implementation
-behind a flag; these tests stay green through that transition because
-they characterise the **Celery chord** code path specifically.
+"""Inventory canary: no Celery ``chord(...)`` anywhere in ``workers/``.
+
+Chord was the fan-out primitive before the PG barrier, and the highest-risk
+Celery construct at our scale (silent task drops at ~130K tasks). This suite
+originally characterised the two inline ``chord(header)(body)`` call sites, then
+narrowed to "exactly one call site, inside ``queue_backend/barrier.py``" once
+they were lifted behind the ``Barrier`` abstraction.
+
+UN-4078 deleted that last call site along with ``CeleryChordBarrier``, so the
+expected count is now **zero**. What is left is worth keeping rather than
+deleting with the rest: chord is still importable from the Celery library that
+the PG consumer itself is built on, so nothing at the type or import level stops
+someone reintroducing one. A reintroduced chord would publish to a broker with
+no consumers, and — unlike a bad PG dispatch — would fail silently, with no
+``pg_barrier_state`` row for the reaper to recover from.
+
+The behavioural characterisation tests that used to live here were deleted with
+the code they mocked: they patched ``queue_backend.barrier.chord``, which no
+longer exists. The fan-out contract they covered (zero batches → ``None``,
+non-empty → enqueue, failures re-raise) is covered against the surviving
+substrate in ``test_pg_barrier.py`` and ``test_barrier.py``.
"""
-from unittest.mock import MagicMock, patch
+import pathlib
+import re
import pytest
+# Top-level directories under workers/ that are not production source.
+_SKIP_TOP_DIRS = {"tests", "__pycache__", "htmlcov", ".venv"}
-# --- Site 1: WorkflowOrchestrationUtils.create_chord_execution ---
-
-
-class TestCreateChordExecution:
- """Characterise the centralised chord helper.
-
- This is the easily-testable site. PR #13 should route both this helper
- AND the inline call (Site 2) through the new Barrier abstraction.
- """
-
- def _make_app_instance(self):
- """Build a Celery app-shaped mock with a working ``.signature(...)``."""
- app = MagicMock()
- # signature() returns an opaque "signature" object — represent it as a
- # MagicMock that we can assert on by identity later.
- app.signature.return_value = MagicMock(name="callback_signature")
- return app
-
- def test_empty_batch_tasks_returns_none_and_skips_chord(self):
- """Zero files: helper short-circuits with None and never calls chord()."""
- from shared.workflow.execution.orchestration_utils import (
- WorkflowOrchestrationUtils,
- )
-
- app = self._make_app_instance()
-
- with patch("queue_backend.barrier.chord") as mock_chord:
- result = WorkflowOrchestrationUtils.create_chord_execution(
- batch_tasks=[],
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-1", "pipeline_id": "pipe-1"},
- callback_queue="file_processing_callback",
- app_instance=app,
- )
-
- assert result is None
- mock_chord.assert_not_called()
-
- def test_non_empty_batch_tasks_invokes_chord(self):
- """Non-empty batch: chord(batch_tasks)(callback_signature) is called."""
- from shared.workflow.execution.orchestration_utils import (
- WorkflowOrchestrationUtils,
- )
-
- app = self._make_app_instance()
- batch_tasks = [MagicMock(name="batch_task_1"), MagicMock(name="batch_task_2")]
-
- with patch("queue_backend.barrier.chord") as mock_chord:
- # chord(batch_tasks) returns a chord object; calling it with the
- # callback signature returns the chord result. Both calls must
- # happen for this characterisation.
- chord_obj = MagicMock(name="chord_object")
- mock_chord.return_value = chord_obj
-
- WorkflowOrchestrationUtils.create_chord_execution(
- batch_tasks=batch_tasks,
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="file_processing_callback",
- app_instance=app,
- )
-
- # First call: chord(batch_tasks)
- mock_chord.assert_called_once_with(batch_tasks)
- # Second call: chord_obj(callback_signature) — applies the chord
- chord_obj.assert_called_once_with(app.signature.return_value)
-
- def test_callback_signature_is_built_with_correct_kwargs(self):
- """The callback signature must be created with the exact task name,
- kwargs, and queue passed in. PR #13 must preserve these."""
- from shared.workflow.execution.orchestration_utils import (
- WorkflowOrchestrationUtils,
- )
-
- app = self._make_app_instance()
- batch_tasks = [MagicMock(name="batch")]
- callback_kwargs = {
- "execution_id": "exec-42",
- "pipeline_id": "pipe-7",
- "organization_id": "org-x",
- }
-
- with patch("queue_backend.barrier.chord") as mock_chord:
- mock_chord.return_value = MagicMock()
- WorkflowOrchestrationUtils.create_chord_execution(
- batch_tasks=batch_tasks,
- callback_task_name="process_batch_callback_api",
- callback_kwargs=callback_kwargs,
- callback_queue="api_file_processing_callback",
- app_instance=app,
- )
-
- app.signature.assert_called_once_with(
- "process_batch_callback_api",
- kwargs=callback_kwargs,
- queue="api_file_processing_callback",
- )
-
- def test_returns_chord_result_object(self):
- """The helper must return whatever ``chord(...)(...)`` returns —
- callers depend on this return value to track the chord."""
- from shared.workflow.execution.orchestration_utils import (
- WorkflowOrchestrationUtils,
- )
-
- app = self._make_app_instance()
-
- with patch("queue_backend.barrier.chord") as mock_chord:
- chord_obj = MagicMock()
- chord_result = MagicMock(name="chord_result_object")
- chord_obj.return_value = chord_result
- mock_chord.return_value = chord_obj
-
- result = WorkflowOrchestrationUtils.create_chord_execution(
- batch_tasks=[MagicMock()],
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="file_processing_callback",
- app_instance=app,
- )
-
- assert result is chord_result
-
- def test_chord_failure_is_re_raised_after_logging(self):
- """If chord() raises, the helper logs and re-raises (not swallowed)."""
- from shared.workflow.execution.orchestration_utils import (
- WorkflowOrchestrationUtils,
- )
-
- app = self._make_app_instance()
-
- with patch("queue_backend.barrier.chord") as mock_chord:
- mock_chord.side_effect = RuntimeError("broker exploded")
- with pytest.raises(RuntimeError, match="broker exploded"):
- WorkflowOrchestrationUtils.create_chord_execution(
- batch_tasks=[MagicMock()],
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="file_processing_callback",
- app_instance=app,
- )
+def _production_py_files() -> list[pathlib.Path]:
+ workers_root = pathlib.Path(__file__).parent.parent
+ out = []
+ for py in workers_root.rglob("*.py"):
+ rel_parts = py.relative_to(workers_root).parts
+ if rel_parts and rel_parts[0] in _SKIP_TOP_DIRS:
+ continue
+ out.append(py)
+ return out
-# --- Site 1 (mixin wrapper): WorkflowOrchestrationMixin.create_chord ---
-
+def _scan(pattern: re.Pattern, *, skip_comments: bool) -> list[str]:
+ workers_root = pathlib.Path(__file__).parent.parent
+ hits = []
+ for py in _production_py_files():
+ for line_no, line in enumerate(py.read_text().splitlines(), start=1):
+ if skip_comments and line.lstrip().startswith("#"):
+ continue
+ if pattern.search(line):
+ hits.append(f"{py.relative_to(workers_root)}:{line_no}")
+ return hits
-class TestWorkflowOrchestrationMixinCreateChord:
- """Characterise the mixin wrapper around ``create_chord_execution``.
- The mixin adds two unique behaviours over the static helper:
- 1. Extracts ``self.app`` from the task context.
- 2. Raises ``RuntimeError`` when no app is bound to the task.
+class TestNoChordRemains:
+ """``chord(...)`` must not reappear in production worker code."""
- Both must be preserved by PR #13 when it lifts the mixin to use Barrier.
- """
+ def test_no_chord_invocation(self):
+ """No assignment-form ``= chord(`` call anywhere in workers/.
- def test_create_chord_extracts_app_from_self_and_delegates(self):
- """The mixin must read ``self.app`` and forward it to the static helper."""
- from shared.workflow.execution.orchestration_utils import (
- WorkflowOrchestrationMixin,
- WorkflowOrchestrationUtils,
+ **Known blind spots**, unchanged from when this asserted a count of one:
+ a return-value-discarded ``chord(batch)(cb)`` evades the assignment-form
+ regex, and an aliased import (``from celery import chord as c``) or
+ module-attribute access (``import celery`` + ``celery.chord(...)``)
+ evades both this and the import canary below. Those shapes are
+ non-idiomatic and appear nowhere in the tree; broaden these canaries if
+ one ever ships.
+ """
+ hits = _scan(re.compile(r"=\s*chord\("), skip_comments=True)
+ assert hits == [], (
+ "Celery chord(...) call site(s) found in workers/:\n "
+ + "\n ".join(hits)
+ + "\nChord was removed with the Celery transport (UN-4078). A chord "
+ "publishes to a broker that has no consumers, and fails silently — "
+ "no pg_barrier_state row is written, so the reaper cannot recover "
+ "the stranded execution. Fan out through "
+ "WorkflowOrchestrationUtils.create_chord_execution (PgBarrier)."
)
- # Build a synthetic task-like object carrying an `app` attribute.
- task = type("FakeTask", (WorkflowOrchestrationMixin,), {})()
- task.app = MagicMock(name="celery_app")
- task.app.signature.return_value = MagicMock(name="callback_signature")
-
- with patch.object(
- WorkflowOrchestrationUtils, "create_chord_execution"
- ) as mock_static:
- mock_static.return_value = MagicMock(name="chord_result")
- batch = [MagicMock()]
- kwargs = {"execution_id": "exec-mixin"}
- task.create_chord(
- batch_tasks=batch,
- callback_task_name="process_batch_callback",
- callback_kwargs=kwargs,
- callback_queue="file_processing_callback",
- )
-
- # Mixin now forwards ``fairness=`` (None when caller omits it)
- # to ``create_chord_execution``. The positional args stay the
- # same.
- mock_static.assert_called_once_with(
- batch,
- "process_batch_callback",
- kwargs,
- "file_processing_callback",
- task.app,
- fairness=None,
+ def test_no_chord_import(self):
+ """``from celery import chord`` must not appear in workers/."""
+ hits = _scan(
+ re.compile(r"^\s*from\s+celery\s+import\s+.*\bchord\b"), skip_comments=False
)
-
- def test_create_chord_raises_when_no_app_bound(self):
- """No ``self.app`` (e.g., called outside a Celery task context):
- the mixin must raise ``RuntimeError`` rather than silently failing
- or passing ``None`` downstream to chord()."""
- from shared.workflow.execution.orchestration_utils import (
- WorkflowOrchestrationMixin,
+ assert hits == [], (
+ "`from celery import chord` found in workers/:\n " + "\n ".join(hits)
)
- task = type("FakeTask", (WorkflowOrchestrationMixin,), {})()
- # Deliberately do NOT set task.app — leave it absent.
-
- with pytest.raises(RuntimeError, match="Celery app instance not available"):
- task.create_chord(
- batch_tasks=[MagicMock()],
- callback_task_name="process_batch_callback",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="file_processing_callback",
- )
-
-
-# --- Cross-site invariant: chord call-site inventory ---
-
-
-class TestChordSiteInventory:
- """Post-Barrier-uplift invariant: exactly ONE ``chord(...)`` call
- site, inside ``queue_backend/barrier.py``.
-
- Before the uplift there were two inline ``chord(...)`` calls (in
- ``orchestration_utils.py`` and ``api-deployment/tasks.py``). The
- refactor lifted both behind ``CeleryChordBarrier`` so the chord
- primitive has a single home — a future ``RedisDecrBarrier`` swap
- can target one location instead of chasing call sites.
-
- If a third ``chord(...)`` call appears anywhere outside
- ``barrier.py``, this test fails — preventing a silent regression
- where a future PR bypasses the abstraction.
- """
-
- _CHORD_HOST = "queue_backend/barrier.py"
-
- def test_chord_invocation_lives_only_in_barrier(self):
- """Count chord(...) invocations (not imports) in workers/ source.
-
- Tightened regex (after the original ``(?:^|[\\s=(])chord\\(``
- could match docstring prose mentioning ``chord(...)`` after
- whitespace): now requires assignment form ``= chord(`` (with
- optional whitespace), which matches the production call at
- ``barrier.py``'s ``result = chord(header_tasks)(callback_signature)``
- but rejects docstring / comment mentions like ``the chord(...)
- primitive``.
+ def test_scanner_sees_a_realistic_tree(self):
+ """Non-vacuity lock: the scan must actually walk production files.
- **Known blind spot**: a return-value-discarded ``chord(batch)(cb)``
- at column 0 (no assignment) would slip past this regex AND the
- comment-line filter. The companion ``test_chord_import_only_in_barrier``
- canary catches the common ``from celery import chord`` import
- form — so a stray call site reachable via that import would
- still trip it. However, aliased imports (``from celery import
- chord as c``) or module-attribute access (``import celery`` +
- ``celery.chord(...)``) would evade *both* canaries. We accept
- this gap because (a) those import shapes are non-idiomatic and
- not used anywhere in workers/, and (b) ``barrier.py`` is the
- sole sanctioned chord call site in the codebase, so any stray
- call would surface in code review long before runtime. If a
- future PR ships either of those import shapes outside
- ``barrier.py``, the canaries need broadening.
+ Without this, a path bug (wrong root, over-broad skip list) would make
+ both canaries above pass over an empty set and report all-clear forever
+ — the exact failure mode a zero-expectation assertion invites.
"""
- import pathlib
- import re
-
- workers_root = pathlib.Path(__file__).parent.parent
- # Anchor to the top-level directory relative to workers_root so we
- # don't accidentally exclude legitimately-named subdirectories like
- # `workers/shared/tests_helpers/`.
- skip_top_dirs = {"tests", "__pycache__", "htmlcov", ".venv"}
-
- # Assignment-form ``= chord(`` — production call sites all
- # capture the AsyncResult into a variable. Docstring prose
- # never uses this form.
+ files = _production_py_files()
+ assert len(files) > 50, f"expected a populated tree, walked {len(files)} files"
+ names = {f.name for f in files}
+ assert "pg_barrier.py" in names
+ assert "barrier.py" in names
+
+ def test_detector_matches_a_known_bad_line(self):
+ """Positive-detection lock for the invocation regex."""
pattern = re.compile(r"=\s*chord\(")
-
- hits = []
- for py in workers_root.rglob("*.py"):
- rel_parts = py.relative_to(workers_root).parts
- if rel_parts and rel_parts[0] in skip_top_dirs:
- continue
- text = py.read_text()
- for line_no, line in enumerate(text.splitlines(), start=1):
- stripped = line.lstrip()
- # Skip pure comment lines (belt-and-suspenders — the
- # regex already rejects most of these via the ``=``
- # prefix requirement, but a comment like ``# x = chord(...)``
- # would otherwise sneak through).
- if stripped.startswith("#"):
- continue
- if pattern.search(line):
- hits.append(f"{py.relative_to(workers_root)}:{line_no}")
-
- assert len(hits) == 1, (
- "Expected exactly 1 chord(...) call site in workers/ (inside "
- f"{self._CHORD_HOST}), found {len(hits)}:\n "
- + "\n ".join(hits)
- + "\nPhase 6a lifted both call sites behind CeleryChordBarrier — "
- "a new direct chord(...) call bypasses the Barrier abstraction "
- "and breaks the future Phase 6b RedisDecrBarrier swap point."
- )
- assert self._CHORD_HOST in hits[0], (
- f"chord(...) call must live in {self._CHORD_HOST}, found at {hits[0]}"
- )
-
- def test_chord_import_only_in_barrier(self):
- """`from celery import chord` should appear in exactly one file —
- ``queue_backend/barrier.py`` — post-Barrier uplift."""
- import pathlib
- import re
-
- workers_root = pathlib.Path(__file__).parent.parent
- skip_top_dirs = {"tests", "__pycache__", "htmlcov", ".venv"}
-
- pattern = re.compile(r"^\s*from\s+celery\s+import\s+chord\b")
-
- hits = []
- for py in workers_root.rglob("*.py"):
- rel_parts = py.relative_to(workers_root).parts
- if rel_parts and rel_parts[0] in skip_top_dirs:
- continue
- text = py.read_text()
- for line_no, line in enumerate(text.splitlines(), start=1):
- if pattern.search(line):
- hits.append(f"{py.relative_to(workers_root)}:{line_no}")
-
- assert len(hits) == 1, (
- f"Expected `from celery import chord` in exactly 1 file, found "
- f"{len(hits)}:\n " + "\n ".join(hits)
- )
- assert self._CHORD_HOST in hits[0], (
- f"chord import must live in {self._CHORD_HOST}, found at {hits[0]}"
- )
+ assert pattern.search(" result = chord(header_tasks)(callback)")
+ assert not pattern.search(" # the chord(...) primitive is gone")
if __name__ == "__main__":
diff --git a/workers/tests/test_dispatch_sites_characterisation.py b/workers/tests/test_dispatch_sites_characterisation.py
index 9bf4daad3..e1573c08b 100644
--- a/workers/tests/test_dispatch_sites_characterisation.py
+++ b/workers/tests/test_dispatch_sites_characterisation.py
@@ -16,6 +16,7 @@ drops a field, rewires a queue, or changes an endpoint fails loudly:
from unittest.mock import MagicMock, patch
import pytest
+from queue_backend import QueueBackend
from shared.patterns.notification.helper import (
ENQUEUE_BUFFER_ENDPOINT,
_enqueue_to_buffer,
@@ -219,11 +220,13 @@ class TestSchedulerDispatchSite:
assert args[4] is True # scheduled flag (always True here)
def test_dispatch_kwargs_layout(self):
- """Kwargs MUST contain use_file_history, pipeline_id, and transport.
+ """Kwargs MUST contain exactly use_file_history, pipeline_id and the
+ write-only ``transport`` shim.
- ``transport`` (9e) is carried in the task payload so the pipeline stays
- on one transport end-to-end. It defaults to ``"celery"`` when the
- create-execution response omits it (older backend / inert PR 1).
+ Nothing reads ``transport`` after UN-4078, but a pre-UN-4078 general
+ worker falls back to Celery when it is absent, so it stays written for
+ one release. Asserting the whole dict is deliberate: it fails on a stray
+ key, and it fails if the shim is dropped too early.
"""
from scheduler.tasks import _execute_scheduled_workflow
@@ -237,50 +240,43 @@ class TestSchedulerDispatchSite:
assert kwargs == {
"use_file_history": True,
"pipeline_id": "pipe-007",
- "transport": "celery",
- }
-
- def test_dispatch_carries_backend_resolved_transport(self):
- """The transport the backend returns from create-execution is threaded
- verbatim into the dispatched task's payload (payload-carry, 9e)."""
- from scheduler.tasks import _execute_scheduled_workflow
-
- api = MagicMock()
- api.create_workflow_execution.return_value = {
- "execution_id": "exec-123",
"transport": "pg_queue",
}
- with patch("scheduler.tasks.dispatch") as mock_dispatch:
- _execute_scheduled_workflow(api, self._make_context())
-
- assert mock_dispatch.call_args.kwargs["kwargs"]["transport"] == "pg_queue"
+ def test_dispatch_ignores_a_transport_field_from_the_backend(self):
+ """A ``transport`` value from the backend response is never forwarded.
- def test_pg_transport_routes_dispatch_to_pg_backend(self):
- """The one line that actually routes the scheduled orchestrator onto PG:
- transport=="pg_queue" → dispatch(backend=QueueBackend.PG) (identity, not
- the allow-list)."""
- from queue_backend import QueueBackend
+ The payload carries the constant ``pg_queue`` shim regardless of what the
+ create-execution response says, so a stale ``"celery"`` cannot reach an
+ un-upgraded consumer that still branches on it.
+ """
from scheduler.tasks import _execute_scheduled_workflow
api = MagicMock()
api.create_workflow_execution.return_value = {
"execution_id": "exec-123",
- "transport": "pg_queue",
+ "transport": "celery",
}
+
with patch("scheduler.tasks.dispatch") as mock_dispatch:
_execute_scheduled_workflow(api, self._make_context())
+ assert mock_dispatch.call_args.kwargs["kwargs"]["transport"] == "pg_queue"
assert mock_dispatch.call_args.kwargs["backend"] is QueueBackend.PG
- def test_celery_transport_leaves_backend_none(self):
- """transport=="celery" → backend=None (legacy Celery dispatch unchanged)."""
+ def test_backend_is_always_pg(self):
+ """The scheduled orchestrator always pins the dispatch to PG.
+
+ This used to be conditional on the payload's transport, with ``None``
+ (defer to select_backend) on the Celery branch. Stating it explicitly
+ keeps the whole execution pipeline pinned at its entry point.
+ """
from scheduler.tasks import _execute_scheduled_workflow
with patch("scheduler.tasks.dispatch") as mock_dispatch:
_execute_scheduled_workflow(self._make_api_client(), self._make_context())
- assert mock_dispatch.call_args.kwargs["backend"] is None
+ assert mock_dispatch.call_args.kwargs["backend"] is QueueBackend.PG
def test_no_dispatch_when_execution_creation_fails(self):
"""If api_client.create_workflow_execution returns no execution_id,
diff --git a/workers/tests/test_dispatch_with_callback.py b/workers/tests/test_dispatch_with_callback.py
index dce8b73f9..f183e68bd 100644
--- a/workers/tests/test_dispatch_with_callback.py
+++ b/workers/tests/test_dispatch_with_callback.py
@@ -16,9 +16,11 @@ from executor.executors.constants import (
PromptServiceConstants as PSKeys,
)
from unstract.sdk1.execution.context import ExecutionContext, Operation
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.sdk1.execution.registry import ExecutorRegistry
from unstract.sdk1.execution.result import ExecutionResult
+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher
+
+from .executor_dispatch_fakes import FakeExecutorTransport, callback_signature
# ---------------------------------------------------------------------------
# Patch targets
@@ -176,40 +178,43 @@ def _make_output(name="field_a", prompt="What is the revenue?", **overrides):
class TestDispatchWithCallback:
- """Verify dispatch_with_callback passes link/link_error to send_task."""
+ """Callbacks ride the enqueue payload as continuations, not Celery links.
+
+ PG has no broker mechanism to fire ``link`` / ``link_error``, so the executor
+ consumer self-chains instead: the signatures are translated to serialisable
+ ``ContinuationSpec``s and carried in the message. These assertions therefore
+ check the *payload*, where the old suite checked ``send_task`` kwargs.
+ """
- def test_callback_kwargs_passed(self):
- mock_app = MagicMock()
- mock_app.send_task.return_value = MagicMock(id="task-123")
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
+ def _dispatcher(self):
+ transport = FakeExecutorTransport(result={"success": True, "data": {}})
+ return PgExecutionDispatcher(transport), transport
+ def test_callback_continuations_and_task_id_passed(self):
+ dispatcher, transport = self._dispatcher()
ctx = ExecutionContext(
executor_name="legacy",
operation="answer_prompt",
run_id="run-cb-1",
execution_source="ide",
)
- on_success = MagicMock(name="success_sig")
- on_error = MagicMock(name="error_sig")
-
result = dispatcher.dispatch_with_callback(
ctx,
- on_success=on_success,
- on_error=on_error,
+ on_success=callback_signature("cb.success"),
+ on_error=callback_signature("cb.error"),
task_id="pre-generated-id",
)
- call_kwargs = mock_app.send_task.call_args
- assert call_kwargs.kwargs["link"] is on_success
- assert call_kwargs.kwargs["link_error"] is on_error
- assert call_kwargs.kwargs["task_id"] == "pre-generated-id"
- assert result.id == "task-123"
-
- def test_no_callbacks_omits_link_kwargs(self):
- mock_app = MagicMock()
- mock_app.send_task.return_value = MagicMock(id="task-456")
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
+ call = transport.only_call
+ assert call["on_success"]["task_name"] == "cb.success"
+ assert call["on_error"]["task_name"] == "cb.error"
+ assert call["task_id"] == "pre-generated-id"
+ # The handle echoes the caller's task_id — on Celery this came back off
+ # the AsyncResult, so a caller reading ``.id`` is unaffected by the move.
+ assert result.id == "pre-generated-id"
+ def test_no_callbacks_leaves_continuations_unset(self):
+ dispatcher, transport = self._dispatcher()
ctx = ExecutionContext(
executor_name="legacy",
operation="extract",
@@ -218,25 +223,13 @@ class TestDispatchWithCallback:
)
dispatcher.dispatch_with_callback(ctx)
- call_kwargs = mock_app.send_task.call_args
- assert "link" not in call_kwargs.kwargs
- assert "link_error" not in call_kwargs.kwargs
-
- def test_no_app_raises(self):
- dispatcher = ExecutionDispatcher(celery_app=None)
- ctx = ExecutionContext(
- executor_name="legacy",
- operation="extract",
- run_id="run-cb-3",
- execution_source="tool",
- )
- with pytest.raises(ValueError, match="No Celery app"):
- dispatcher.dispatch_with_callback(ctx)
+ call = transport.only_call
+ assert call["on_success"] is None
+ assert call["on_error"] is None
-
-# ---------------------------------------------------------------------------
-# 5C: ide_index compound operation through eager chain
-# ---------------------------------------------------------------------------
+ # ``test_no_app_raises`` is gone with the Celery dispatcher: PG dispatch is
+ # transport-injected, so there is no "no app configured" state to raise on.
+ # The enqueue-failure contract it stood in for lives in test_executor_rpc.py.
class TestIdeIndexEagerChain:
@@ -962,18 +955,14 @@ class TestOperationEnum:
class TestDispatcherModes:
- """Verify all three dispatch modes work."""
+ """Verify all three dispatch modes work on the PG transport."""
def test_dispatch_sync(self):
- """dispatch() calls send_task and .get()."""
- mock_app = MagicMock()
- async_result = MagicMock()
- async_result.get.return_value = ExecutionResult(
- success=True, data={"test": 1}
- ).to_dict()
- mock_app.send_task.return_value = async_result
-
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
+ """dispatch() enqueues once and blocks for the result row."""
+ transport = FakeExecutorTransport(
+ result=ExecutionResult(success=True, data={"test": 1}).to_dict()
+ )
+ dispatcher = PgExecutionDispatcher(transport)
ctx = ExecutionContext(
executor_name="legacy",
operation="extract",
@@ -983,15 +972,13 @@ class TestDispatcherModes:
result = dispatcher.dispatch(ctx, timeout=10)
assert result.success
- mock_app.send_task.assert_called_once()
- async_result.get.assert_called_once()
+ assert len(transport.enqueue_calls) == 1
+ assert transport.wait_timeouts == [10]
def test_dispatch_async(self):
- """dispatch_async() returns task_id without blocking."""
- mock_app = MagicMock()
- mock_app.send_task.return_value = MagicMock(id="async-id")
-
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
+ """dispatch_async() returns the task id it enqueued, without blocking."""
+ transport = FakeExecutorTransport()
+ dispatcher = PgExecutionDispatcher(transport)
ctx = ExecutionContext(
executor_name="legacy",
operation="extract",
@@ -1000,5 +987,8 @@ class TestDispatcherModes:
)
task_id = dispatcher.dispatch_async(ctx)
- assert task_id == "async-id"
- mock_app.send_task.assert_called_once()
+ # PG mints the id itself (the Celery path read it off the AsyncResult),
+ # so assert it round-trips onto the message rather than pinning a literal.
+ assert task_id
+ assert transport.only_call["task_id"] == task_id
+ assert transport.wait_timeouts == []
diff --git a/workers/tests/test_executor_dispatch.py b/workers/tests/test_executor_dispatch.py
index e57e329db..385dae472 100644
--- a/workers/tests/test_executor_dispatch.py
+++ b/workers/tests/test_executor_dispatch.py
@@ -1,22 +1,20 @@
-"""Tests for the file_processing → executor boundary: header forwarding
-through ExecutionDispatcher and inventory canary for raw send_task calls.
+"""Tests for the file_processing → executor boundary: payload/queue routing
+through the PG executor dispatcher, and an inventory canary for raw send_task calls.
"""
from __future__ import annotations
import ast
from typing import Any
-from unittest.mock import MagicMock
import pytest
-from celery import Celery
-
from queue_backend import FairnessKey
from queue_backend.fairness import WorkloadType
from unstract.sdk1.execution.context import ExecutionContext, Operation
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher
from .canary_helpers import iter_production_trees
+from .executor_dispatch_fakes import FakeExecutorTransport, callback_signature
# Promote ``UserWarning`` from ``iter_production_trees`` (emitted on
# unparseable production files) to a test failure. Without this an
@@ -43,85 +41,71 @@ def _make_context(**overrides: Any) -> ExecutionContext:
return ExecutionContext(**defaults)
-class TestExecutionDispatcherForwardsHeaders:
- """``ExecutionDispatcher`` propagates the ``headers`` kwarg to Celery."""
+class TestPgDispatchCarriesOrgInPayload:
+ """Org routing rides the enqueue payload, and ``headers=`` is rejected.
+
+ This replaces the old ``ExecutionDispatcher`` header-forwarding suite. That
+ contract is deliberately gone, not merely relocated: the PG dispatch methods
+ take **no** ``headers`` argument and carry org/routing in the payload instead
+ (``transport.enqueue(..., org_id=...)``). Locking the absence matters —
+ three call sites kept passing ``headers=`` after the routing dispatcher was
+ removed in UN-4046 and every extraction raised ``TypeError`` until they were
+ found. A test that only checked the happy path would not have caught it.
+ """
- def _patched_app(self) -> tuple[ExecutionDispatcher, MagicMock]:
- app = MagicMock(spec=Celery)
- async_result = MagicMock()
- async_result.id = "task-1"
- async_result.get.return_value = {"success": True, "data": {}}
- app.send_task.return_value = async_result
- return ExecutionDispatcher(celery_app=app), app
+ def _dispatcher(self) -> tuple[PgExecutionDispatcher, FakeExecutorTransport]:
+ transport = FakeExecutorTransport(result={"success": True, "data": {}})
+ return PgExecutionDispatcher(transport), transport
- # The three dispatch entry points share the same header-forwarding
- # contract via ``_build_send_kwargs``; parametrize over them so a
- # divergence (e.g. one method dropping ``headers=``) surfaces with
- # per-method failure granularity via the parametrize IDs.
@pytest.mark.parametrize(
"method", ["dispatch", "dispatch_async", "dispatch_with_callback"]
)
- def test_forwards_headers(self, method):
- # Use ``FairnessKey.as_header()`` as the fixture rather than
- # hand-built dicts so the test exercises the exact wire shape
- # real producers emit (including ``pipeline_priority``).
- headers = FairnessKey(
- org_id="org-1", workload_type=WorkloadType.NON_API
- ).as_header()
- d, app = self._patched_app()
- getattr(d, method)(_make_context(), headers=headers)
- assert app.send_task.call_args.kwargs["headers"] == headers
+ def test_org_id_travels_in_the_payload(self, method):
+ d, transport = self._dispatcher()
+ getattr(d, method)(_make_context(organization_id="org-1"))
+ assert transport.only_call["org_id"] == "org-1"
@pytest.mark.parametrize(
"method", ["dispatch", "dispatch_async", "dispatch_with_callback"]
)
- def test_omits_headers_when_none(self, method):
- # Caller passes no headers ⇒ ``headers`` kwarg not forwarded
- # to send_task (preserves the call shape Celery's link /
- # link_error handling expects).
- d, app = self._patched_app()
- getattr(d, method)(_make_context())
- assert "headers" not in app.send_task.call_args.kwargs
+ def test_no_headers_kwarg_accepted(self, method):
+ """Passing ``headers=`` must fail loudly, not be silently absorbed."""
+ d, _ = self._dispatcher()
+ with pytest.raises(TypeError):
+ getattr(d, method)(_make_context(), headers={"x-fairness-key": {}})
@pytest.mark.parametrize(
"method", ["dispatch", "dispatch_async", "dispatch_with_callback"]
)
- def test_omits_headers_when_empty_dict(self, method):
- """Empty header dicts are dropped (treated as a no-headers
- call). ``FairnessKey.as_header()`` can never legitimately
- return ``{}`` — forwarding it would document a producer-side
- build bug rather than catch it.
- """
- d, app = self._patched_app()
- getattr(d, method)(_make_context(), headers={})
- assert "headers" not in app.send_task.call_args.kwargs
-
- def test_dispatch_with_callback_combines_headers_and_callbacks(self):
- """All four optional kwargs (headers, on_success, on_error,
- task_id) land on the same ``send_task`` call. A merge bug in
- ``_build_send_kwargs`` would slip through the single-kwarg
- forwarding tests above.
+ def test_queue_derives_from_executor_name(self, method):
+ d, transport = self._dispatcher()
+ getattr(d, method)(_make_context(executor_name="table"))
+ assert transport.queue == "celery_executor_table"
+
+ def test_dispatch_with_callback_carries_continuations_and_task_id(self):
+ """Callbacks ride the payload as continuations, not Celery link kwargs.
+
+ The signatures are built with a real name/queue and no positional args
+ because ``signature_to_continuation`` rejects all three otherwise — PG
+ self-chaining routes by the row's queue and supports kwargs-only
+ callbacks. A bare ``MagicMock`` passes none of those checks, so shaping
+ them here is what makes the assertion meaningful.
"""
- from celery.canvas import Signature
-
- d, app = self._patched_app()
- headers = FairnessKey(
- org_id="org-1", workload_type=WorkloadType.NON_API
- ).as_header()
- on_success = MagicMock(spec=Signature)
- on_error = MagicMock(spec=Signature)
- d.dispatch_with_callback(
+ d, transport = self._dispatcher()
+ on_success = callback_signature("cb.success", queue="celery_callback")
+ on_error = callback_signature("cb.error", queue="celery_callback")
+ handle = d.dispatch_with_callback(
_make_context(),
on_success=on_success,
on_error=on_error,
task_id="t-1",
- headers=headers,
)
- kwargs = app.send_task.call_args.kwargs
- assert kwargs["headers"] == headers
- assert kwargs["link"] is on_success
- assert kwargs["link_error"] is on_error
- assert kwargs["task_id"] == "t-1"
+ call = transport.only_call
+ assert call["task_id"] == "t-1"
+ assert handle.id == "t-1"
+ assert call["on_success"]["task_name"] == "cb.success"
+ assert call["on_error"]["task_name"] == "cb.error"
+ assert "link" not in call and "link_error" not in call
class TestFairnessKeyComposesWithHeaders:
@@ -154,7 +138,7 @@ class TestFairnessKeyComposesWithHeaders:
class TestExecuteExtractionDispatchInventory:
"""Canary: ``execute_extraction`` must only be dispatched via
- ``ExecutionDispatcher``. Raw **string-literal**
+ ``PgExecutionDispatcher``. Raw **string-literal**
``*.send_task("execute_extraction", ...)`` elsewhere is forbidden.
Known blind spots (deliberate — widening adds AST resolution cost
@@ -174,9 +158,10 @@ class TestExecuteExtractionDispatchInventory:
]
assert offenders == [], (
"Production code calls ``*.send_task(\"execute_extraction\", ...)`` "
- "outside ``ExecutionDispatcher``. Use "
- "``ExecutionDispatcher.dispatch(...)`` instead so fairness "
- "headers and queue routing stay consistent. (Detection "
+ "outside ``PgExecutionDispatcher``. Use "
+ "``get_executor_dispatcher().dispatch(...)`` instead so queue routing "
+ "and the reply-key contract stay consistent — and because a raw "
+ "send_task publishes to a broker nothing drains. (Detection "
"covers string-literal task names only; constant references, "
"f-strings, and apply_async are blind spots.) Found:\n "
+ "\n ".join(offenders)
diff --git a/workers/tests/test_extraction_pipeline_contracts.py b/workers/tests/test_extraction_pipeline_contracts.py
index 4eecb2b44..53e2fab08 100644
--- a/workers/tests/test_extraction_pipeline_contracts.py
+++ b/workers/tests/test_extraction_pipeline_contracts.py
@@ -698,20 +698,22 @@ class TestSanityResponseContracts:
class TestSanityDispatcher:
- """Full-chain dispatcher tests with Celery eager mode."""
+ """Full-chain dispatcher tests, running the executor task in-process."""
@patch(_PATCH_FS)
@patch(_PATCH_X2TEXT)
def test_dispatcher_dispatch_full_chain(
self, mock_x2text_cls, mock_get_fs, eager_app
):
- """ExecutionDispatcher dispatches through Celery and returns result.
+ """The dispatcher drives the executor task and returns its result.
- Celery's ``send_task`` doesn't reliably use eager mode, so we
- patch it to route through ``task.apply()`` instead — this still
- exercises the full Dispatcher → task → orchestrator chain.
+ ``EagerExecutorTransport`` runs ``execute_extraction`` in-process rather
+ than enqueueing it, so this still exercises the full
+ dispatcher → task → orchestrator chain without a queue or a database.
"""
- from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
+ from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher
+
+ from .executor_dispatch_fakes import EagerExecutorTransport
mock_x2text = MagicMock()
mock_x2text.process.return_value = _mock_process_response("dispatched")
@@ -719,29 +721,21 @@ class TestSanityDispatcher:
mock_x2text_cls.return_value = mock_x2text
mock_get_fs.return_value = MagicMock()
- task = eager_app.tasks["execute_extraction"]
-
- def eager_send_task(name, args=None, **kwargs):
- return task.apply(args=args)
-
- with patch.object(eager_app, "send_task", side_effect=eager_send_task):
- dispatcher = ExecutionDispatcher(celery_app=eager_app)
- ctx = _extract_ctx()
- result = dispatcher.dispatch(ctx, timeout=10)
+ transport = EagerExecutorTransport(eager_app.tasks["execute_extraction"])
+ dispatcher = PgExecutionDispatcher(transport)
+ result = dispatcher.dispatch(_extract_ctx(), timeout=10)
assert isinstance(result, ExecutionResult)
assert result.success is True
assert result.data[IKeys.EXTRACTED_TEXT] == "dispatched"
-
- def test_dispatcher_no_app_raises(self):
- """ExecutionDispatcher(celery_app=None).dispatch() → ValueError."""
- from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
-
- dispatcher = ExecutionDispatcher(celery_app=None)
- ctx = _extract_ctx()
-
- with pytest.raises(ValueError, match="No Celery app"):
- dispatcher.dispatch(ctx)
+ assert transport.queue == "celery_executor_legacy"
+
+ # ``test_dispatcher_no_app_raises`` is gone with the Celery dispatcher it
+ # characterised: ``PgExecutionDispatcher`` is transport-injected and never
+ # raises by contract — an enqueue failure becomes ``ExecutionResult.failure``.
+ # That contract is covered in ``test_executor_rpc.py``, which drives a
+ # transport whose ``enqueue`` raises; duplicating it here would assert the
+ # shared dispatcher's behaviour a second time in a suite about the pipeline.
class TestSanityCrossCutting:
diff --git a/workers/tests/test_legacy_transport_shim.py b/workers/tests/test_legacy_transport_shim.py
new file mode 100644
index 000000000..034ff6ac6
--- /dev/null
+++ b/workers/tests/test_legacy_transport_shim.py
@@ -0,0 +1,42 @@
+"""Gating tests for the UN-4078 rolling-deploy shim.
+
+Nothing reads the ``transport`` key after UN-4078, so only a test stops it being
+dropped before the follow-up release — and dropping it strands executions that a
+pre-UN-4078 worker picks up during a rolling deploy (it defaults an absent field
+to "celery" and publishes to a broker with no consumers).
+
+These must run in the DEFAULT lane: the ``PgBarrier.enqueue`` tests that also
+cover the descriptor need a real Postgres and are skipped wherever one is absent,
+so they cannot be the only guard. Delete this file with the shim's writes.
+"""
+
+from unstract.core.data_models import LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUE
+
+from queue_backend.pg_barrier import build_callback_descriptor
+
+
+class TestCallbackDescriptorShim:
+ def _descriptor(self):
+ return build_callback_descriptor(
+ task_name="process_batch_callback",
+ kwargs={"execution_id": "exec-1", "organization_id": "org-1"},
+ queue="file_processing_callback",
+ fairness_headers=None,
+ )
+
+ def test_descriptor_carries_the_transport_shim(self):
+ # A pre-UN-4078 consumer reads callback_descriptor.get("transport") and
+ # fires the callback through Celery when it is missing.
+ assert self._descriptor()[LEGACY_TRANSPORT_KEY] == LEGACY_TRANSPORT_VALUE
+
+ def test_shim_value_is_never_celery(self):
+ # "celery" is the one value that must never be written: it selects the
+ # dead transport explicitly rather than by omission.
+ assert self._descriptor()[LEGACY_TRANSPORT_KEY] == "pg_queue"
+
+ def test_descriptor_keeps_its_other_wire_keys(self):
+ d = self._descriptor()
+ assert d["task_name"] == "process_batch_callback"
+ assert d["queue"] == "file_processing_callback"
+ assert d["kwargs"]["execution_id"] == "exec-1"
+ assert d["fairness_headers"] is None
diff --git a/workers/tests/test_orchestration_idempotency.py b/workers/tests/test_orchestration_idempotency.py
index 4768ad583..d31f685b3 100644
--- a/workers/tests/test_orchestration_idempotency.py
+++ b/workers/tests/test_orchestration_idempotency.py
@@ -29,29 +29,23 @@ from queue_backend.pg_barrier import (
)
from queue_backend.pg_queue.connection import create_pg_connection
-# --- Layer 1: transport-gated gate (no DB) ---
+# --- Layer 1: the duplicate-orchestration gate (no DB) ---
class TestOrchestrationGate:
- """``_should_skip_duplicate_orchestration`` — Celery never claims; on PG the
- first delivery proceeds and a duplicate skips.
+ """``_should_skip_duplicate_orchestration`` — the first delivery proceeds and
+ a duplicate skips. The gate used to be transport-conditional (a no-op on
+ Celery, which had no redelivery); with PG the only transport it always runs.
"""
- def test_celery_transport_never_claims(self):
- # The whole guard is PG-only: on Celery it must not even consult the claim
- # (proves the Celery path is behaviorally untouched).
- with patch("general.tasks.try_claim_orchestration") as claim:
- assert _should_skip_duplicate_orchestration("e1", "org1", "celery") is False
- claim.assert_not_called()
-
def test_pg_first_delivery_proceeds(self):
with patch("general.tasks.try_claim_orchestration", return_value=True) as claim:
- assert _should_skip_duplicate_orchestration("e1", "org1", "pg_queue") is False
+ assert _should_skip_duplicate_orchestration("e1", "org1") is False
claim.assert_called_once_with("e1", "org1") # org stamped for the reaper
def test_pg_duplicate_delivery_skips(self):
with patch("general.tasks.try_claim_orchestration", return_value=False):
- assert _should_skip_duplicate_orchestration("e1", "org1", "pg_queue") is True
+ assert _should_skip_duplicate_orchestration("e1", "org1") is True
def test_claim_error_propagates(self):
# A DB error in the claim must PROPAGATE, not be swallowed — a swallow-and-
@@ -62,7 +56,7 @@ class TestOrchestrationGate:
side_effect=psycopg2.OperationalError("db down"),
):
with pytest.raises(psycopg2.OperationalError):
- _should_skip_duplicate_orchestration("e1", "org1", "pg_queue")
+ _should_skip_duplicate_orchestration("e1", "org1")
def test_skip_on_retry_logs_error_with_errorid(self, caplog):
# A skip when retries > 0 is a SUPPRESSED retry (a prior attempt claimed
@@ -71,7 +65,7 @@ class TestOrchestrationGate:
with patch("general.tasks.try_claim_orchestration", return_value=False):
with caplog.at_level(logging.ERROR, logger="general.tasks"):
assert (
- _should_skip_duplicate_orchestration("e1", "org1", "pg_queue", 2)
+ _should_skip_duplicate_orchestration("e1", "org1", 2)
is True
)
assert "ORCH_CLAIM_SUPPRESSED_RETRY" in caplog.text
@@ -105,7 +99,6 @@ class TestOrchestratorShortCircuit:
workflow_id="wf",
execution_id="e-dup",
hash_values_of_files={},
- transport="pg_queue",
)
assert out == {
"status": "skipped_duplicate_orchestration",
@@ -142,7 +135,6 @@ class TestOrchestratorShortCircuit:
workflow_id="wf",
execution_id="e-x",
hash_values_of_files={},
- transport="pg_queue",
)
assert released == [] # claim never acquired → nothing released
@@ -168,7 +160,6 @@ class TestOrchestratorShortCircuit:
workflow_id="wf",
execution_id="e-y",
hash_values_of_files={},
- transport="pg_queue",
)
assert released == ["e-y"] # claimed then failed → released
@@ -205,49 +196,16 @@ class TestOrchestratorShortCircuit:
workflow_id="wf",
execution_id="e-term",
hash_values_of_files={},
- transport="pg_queue",
)
assert out["status"] == "skipped_terminal_execution"
assert released == [] # claim kept → tombstone re-established
api_client.update_workflow_execution_status.assert_not_called() # never re-armed
- def test_terminal_execution_not_skipped_on_celery(self):
- # Celery has no redelivery, so the terminal guard is is_pg-gated → a no-op.
- # A terminal status on the Celery path must NOT short-circuit: we prove the
- # task runs PAST the guard by making the next step (tool validation, called
- # right after the guard) raise a sentinel. Patched via the task's own
- # globals — the bare-``tasks`` module the worker bootstrap loads, not
- # ``general.tasks`` (the sys.path quirk; a module-attr patch would miss it).
- api_client = MagicMock()
- api_client.get_workflow_execution.return_value = MagicMock(
- success=True,
- data={
- "execution": {
- "status": tasks.ExecutionStatus.COMPLETED.value,
- "workflow_id": "wf",
- }
- },
- )
-
- def _sentinel(**_kwargs):
- raise RuntimeError("reached past the guard")
-
- with patch.dict(
- self._task_globals(), {"validate_workflow_tool_instances": _sentinel}
- ):
- with patch.object(
- tasks.WorkerExecutionContext,
- "setup_execution_context",
- return_value=(MagicMock(), api_client),
- ):
- with pytest.raises(RuntimeError, match="reached past the guard"):
- tasks.async_execute_bin_general(
- schema_name="org",
- workflow_id="wf",
- execution_id="e-term-celery",
- hash_values_of_files={},
- transport="celery",
- )
+ # ``test_terminal_execution_not_skipped_on_celery`` is gone with the
+ # transport: the terminal guard used to be is_pg-gated so the Celery path ran
+ # past it. There is no un-guarded path left — the guard above
+ # (``test_terminal_execution_skips_and_keeps_claim``) is now the only
+ # behaviour.
# --- Layer 2: claim primitive against real Postgres ---
diff --git a/workers/tests/test_pg_barrier.py b/workers/tests/test_pg_barrier.py
index f55408a2f..84bf8093e 100644
--- a/workers/tests/test_pg_barrier.py
+++ b/workers/tests/test_pg_barrier.py
@@ -27,7 +27,7 @@ from queue_backend.pg_barrier import (
_barrier_pg_decrement,
_fire_barrier_callback,
barrier_pg_abort,
- barrier_pg_decr_and_check,
+ _barrier_pg_decrement,
claim_batch,
run_batch_with_barrier,
try_claim_orchestration,
@@ -55,11 +55,14 @@ _CALLBACK = {
def _mock_header_task():
- """A header-task Signature whose clone() records link/link_error/apply_async."""
- cloned = MagicMock(name="cloned_signature")
- task = MagicMock(name="header_signature")
- task.clone.return_value = cloned
- return task, cloned
+ """A header-task Signature for the fan-out, PG-shaped.
+
+ Kept as a two-tuple so the many ``_mock_header_task()[0]`` call sites read
+ unchanged; the second element is now unused (it was the ``.clone()`` mock the
+ Celery ``.link`` path recorded against, and that path is gone).
+ """
+ sig = _pg_header()
+ return sig, None
# --- Layer 1: protocol shape + TTL (no DB) ---
@@ -749,21 +752,40 @@ def _last_progress_age_seconds(conn, execution_id):
class TestPgBarrierEnqueue:
- def test_upsert_creates_row_and_attaches_links(self, barrier_db):
+ def test_upsert_creates_row_and_dispatches_headers(self, barrier_db):
tasks = [_mock_header_task() for _ in range(3)]
- handle = PgBarrier().enqueue(
- [t for t, _ in tasks],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-A"},
- callback_queue="general",
- app_instance=None,
- )
+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
+ handle = PgBarrier().enqueue(
+ [t for t, _ in tasks],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-A"},
+ callback_queue="general",
+ app_instance=None,
+ )
assert handle.id == "exec-A"
assert _row(barrier_db, "exec-A") == (3, [])
- for _, cloned in tasks:
- cloned.link.assert_called_once()
- cloned.link_error.assert_called_once()
- cloned.apply_async.assert_called_once()
+ # One PG dispatch per header, each carrying its barrier context. There is
+ # no .link/.link_error to assert on any more — the consumer decrements
+ # in-body from the injected _barrier_context.
+ assert mock_dispatch.call_count == 3
+ for i, call in enumerate(mock_dispatch.call_args_list):
+ assert call.kwargs["kwargs"]["_barrier_context"]["batch_index"] == i
+
+ def test_callback_descriptor_still_carries_legacy_transport(self, barrier_db):
+ # Rolling-deploy shim (UN-4078): a pre-UN-4078 file-processing pod reads
+ # callback_descriptor["transport"] and fires the callback via Celery when
+ # it is absent. Must stay written until the release after UN-4078.
+ task, _ = _mock_header_task()
+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
+ PgBarrier().enqueue(
+ [task],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-LT"},
+ callback_queue="general",
+ app_instance=None,
+ )
+ ctx = mock_dispatch.call_args.kwargs["kwargs"]["_barrier_context"]
+ assert ctx["callback_descriptor"]["transport"] == "pg_queue"
def test_enqueue_sets_expires_cap_and_fresh_progress(self, barrier_db, monkeypatch):
# enqueue stamps expires_at = now()+ttl (the absolute cap) AND
@@ -799,32 +821,35 @@ class TestPgBarrierEnqueue:
monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: barrier_db)
tasks = [_mock_header_task() for _ in range(3)]
- handle = PgBarrier().enqueue(
- [t for t, _ in tasks],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-HEAL"},
- callback_queue="general",
- app_instance=None,
- )
+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
+ handle = PgBarrier().enqueue(
+ [t for t, _ in tasks],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-HEAL"},
+ callback_queue="general",
+ app_instance=None,
+ )
assert handle.id == "exec-HEAL"
assert dead.closed is True # stale conn discarded by _cursor
assert _row(barrier_db, "exec-HEAL") == (3, []) # row landed once, remaining=N
- for _, cloned in tasks: # every header dispatched exactly once
- cloned.apply_async.assert_called_once()
+ assert mock_dispatch.call_count == 3 # every header dispatched exactly once
- def test_fairness_header_stamped(self, barrier_db):
- task, cloned = _mock_header_task()
- PgBarrier().enqueue(
- [task],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-F"},
- callback_queue="general",
- app_instance=None,
- fairness=FairnessKey(org_id="o", workload_type=WorkloadType.API),
- )
- headers = cloned.set.call_args.kwargs["headers"]
- assert FAIRNESS_HEADER_NAME in headers
+ def test_fairness_forwarded_to_dispatch(self, barrier_db):
+ # Fairness reaches the queue row as a FairnessKey on the dispatch, not as
+ # an AMQP header on a cloned signature (that was the .link path).
+ task, _ = _mock_header_task()
+ fairness = FairnessKey(org_id="o", workload_type=WorkloadType.API)
+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
+ PgBarrier().enqueue(
+ [task],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-F"},
+ callback_queue="general",
+ app_instance=None,
+ fairness=fairness,
+ )
+ assert mock_dispatch.call_args.kwargs["fairness"] is fairness
def test_upsert_overwrites_stale_state(self, barrier_db):
# A prior run left a row at remaining=1 with results; the new enqueue
@@ -889,17 +914,20 @@ class TestPgBarrierEnqueue:
def test_mid_loop_dispatch_failure_deletes_row(self, barrier_db):
good, _ = _mock_header_task()
- bad, bad_cloned = _mock_header_task()
- bad_cloned.apply_async.side_effect = RuntimeError("broker down")
+ bad, _ = _mock_header_task()
barrier = PgBarrier()
- with pytest.raises(RuntimeError):
- barrier.enqueue(
- [good, bad],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-D"},
- callback_queue="general",
- app_instance=None,
- )
+ with patch(
+ "queue_backend.dispatch.dispatch",
+ side_effect=[None, RuntimeError("queue write failed")],
+ ):
+ with pytest.raises(RuntimeError):
+ barrier.enqueue(
+ [good, bad],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-D"},
+ callback_queue="general",
+ app_instance=None,
+ )
assert _row(barrier_db, "exec-D") is None # cleaned up
@@ -917,7 +945,7 @@ def _seed(conn, execution_id, remaining, *, results="[]"):
class TestDecrAndCheck:
def test_pending_decrements_only(self, barrier_db):
_seed(barrier_db, "exec-P", 3)
- out = barrier_pg_decr_and_check(
+ out = _barrier_pg_decrement(
{"f": 1}, execution_id="exec-P", callback_descriptor=_CALLBACK
)
assert out["status"] == "pending"
@@ -940,7 +968,7 @@ class TestDecrAndCheck:
" now() - interval '1 hour')"
)
assert _last_progress_age_seconds(barrier_db, "exec-SL") > 3000 # ~1h stale
- barrier_pg_decr_and_check(
+ _barrier_pg_decrement(
{"f": 1}, execution_id="exec-SL", callback_descriptor=_CALLBACK
)
assert _last_progress_age_seconds(barrier_db, "exec-SL") < 5 # refreshed
@@ -1006,9 +1034,9 @@ class TestDecrAndCheck:
lambda **_k: create_pg_connection(env_prefix="TEST_DB_"),
)
- with patch("celery.current_app.signature") as sig:
- sig.return_value.apply_async.return_value = MagicMock(id="cb-ptb")
- out = barrier_pg_decr_and_check(
+ with patch("queue_backend.dispatch.dispatch") as sig:
+ sig.return_value = MagicMock(id="cb-ptb")
+ out = _barrier_pg_decrement(
{"f": "x"}, execution_id="exec-PTB", callback_descriptor=_CALLBACK
)
@@ -1019,35 +1047,54 @@ class TestDecrAndCheck:
def test_complete_fires_callback_with_aggregated_results(self, barrier_db):
_seed(barrier_db, "exec-C", 1, results='[{"f": "a"}]')
- with patch("celery.current_app.signature") as sig:
- sig.return_value.apply_async.return_value = MagicMock(id="cb-task-1")
- out = barrier_pg_decr_and_check(
+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
+ mock_dispatch.return_value = MagicMock(id="cb-task-1")
+ out = _barrier_pg_decrement(
{"f": "b"}, execution_id="exec-C", callback_descriptor=_CALLBACK
)
assert out["status"] == "complete"
# Callback got the full aggregated list as its first positional arg.
- assert sig.call_args.kwargs["args"] == [[{"f": "a"}, {"f": "b"}]]
+ assert mock_dispatch.call_args.kwargs["args"] == [[{"f": "a"}, {"f": "b"}]]
assert _row(barrier_db, "exec-C") is None # row deleted after dispatch
def test_complete_path_passes_fairness_header(self, barrier_db):
_seed(barrier_db, "exec-FH", 1)
- descriptor = {**_CALLBACK, "fairness_headers": {FAIRNESS_HEADER_NAME: {"o": 1}}}
- with patch("celery.current_app.signature") as sig:
- sig.return_value.apply_async.return_value = MagicMock(id="cb")
- barrier_pg_decr_and_check(
+ # A full header payload: the PG path *parses* it back into a FairnessKey
+ # (the Celery path only forwarded the dict opaquely), so a partial one
+ # would fail on the missing workload_type rather than assert anything.
+ descriptor = {
+ **_CALLBACK,
+ "fairness_headers": {
+ FAIRNESS_HEADER_NAME: {
+ "org_id": "o",
+ "workload_type": "api",
+ "pipeline_priority": 5,
+ }
+ },
+ }
+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
+ mock_dispatch.return_value = MagicMock(id="cb")
+ _barrier_pg_decrement(
{"f": 1}, execution_id="exec-FH", callback_descriptor=descriptor
)
- assert sig.call_args.kwargs["headers"] == {FAIRNESS_HEADER_NAME: {"o": 1}}
+ # Reconstructed into a FairnessKey and handed to the dispatch, so the
+ # callback rides the same org/priority its header tasks did.
+ forwarded = mock_dispatch.call_args.kwargs["fairness"]
+ assert forwarded.org_id == "o"
+ assert forwarded.workload_type is WorkloadType.API
+ assert forwarded.pipeline_priority == 5
def test_callback_dispatch_failure_preserves_row(self, barrier_db):
# The central failure-masking invariant: dispatch happens BEFORE the row
- # is deleted, so an apply_async failure leaves the row in place (reclaimed
- # by expiry) — guards against a delete-before-dispatch reorder.
+ # is deleted, so a callback-dispatch failure leaves the row in place
+ # (reclaimed by expiry) — guards against a delete-before-dispatch reorder.
_seed(barrier_db, "exec-CF", 1)
- with patch("celery.current_app.signature") as sig:
- sig.return_value.apply_async.side_effect = RuntimeError("broker down")
+ with patch(
+ "queue_backend.dispatch.dispatch",
+ side_effect=RuntimeError("queue write failed"),
+ ):
with pytest.raises(RuntimeError):
- barrier_pg_decr_and_check(
+ _barrier_pg_decrement(
{"f": 1}, execution_id="exec-CF", callback_descriptor=_CALLBACK
)
assert _row(barrier_db, "exec-CF") is not None # row survives for TTL reclaim
@@ -1056,7 +1103,7 @@ class TestDecrAndCheck:
# jsonb_build_array() must append a list-shaped result as ONE element
# (plain `||` would concatenate it). Guards that choice.
_seed(barrier_db, "exec-L", 2)
- barrier_pg_decr_and_check(
+ _barrier_pg_decrement(
[1, 2], execution_id="exec-L", callback_descriptor=_CALLBACK
)
remaining, results = _row(barrier_db, "exec-L")
@@ -1068,7 +1115,7 @@ class TestDecrAndCheck:
# be torn down (fail fast) rather than hang to expiry.
_seed(barrier_db, "exec-NB", 1)
with pytest.raises(psycopg2.DataError):
- barrier_pg_decr_and_check(
+ _barrier_pg_decrement(
{"f": "bad\x00value"},
execution_id="exec-NB",
callback_descriptor=_CALLBACK,
@@ -1081,8 +1128,8 @@ class TestDecrAndCheck:
# fires — even when it would otherwise have hit remaining == 0.
_seed(barrier_db, "exec-DA", 1)
barrier_pg_abort(execution_id="exec-DA") # header failed → row deleted
- with patch("celery.current_app.signature") as sig:
- out = barrier_pg_decr_and_check(
+ with patch("queue_backend.dispatch.dispatch") as sig:
+ out = _barrier_pg_decrement(
{"f": 1}, execution_id="exec-DA", callback_descriptor=_CALLBACK
)
assert out["status"] == "abandoned"
@@ -1090,8 +1137,8 @@ class TestDecrAndCheck:
def test_negative_remaining_does_not_fire(self, barrier_db):
_seed(barrier_db, "exec-N", 0) # decrement → -1
- with patch("celery.current_app.signature") as sig:
- out = barrier_pg_decr_and_check(
+ with patch("queue_backend.dispatch.dispatch") as sig:
+ out = _barrier_pg_decrement(
{"f": 1}, execution_id="exec-N", callback_descriptor=_CALLBACK
)
assert out["status"] == "abandoned"
@@ -1099,8 +1146,8 @@ class TestDecrAndCheck:
assert _row(barrier_db, "exec-N") is None
def test_missing_row_does_not_fire(self, barrier_db):
- with patch("celery.current_app.signature") as sig:
- out = barrier_pg_decr_and_check(
+ with patch("queue_backend.dispatch.dispatch") as sig:
+ out = _barrier_pg_decrement(
{"f": 1}, execution_id="nope", callback_descriptor=_CALLBACK
)
assert out["status"] == "abandoned"
@@ -1110,14 +1157,10 @@ class TestDecrAndCheck:
_seed(barrier_db, "exec-U", 1)
bad_result = {object()}
with pytest.raises(TypeError):
- barrier_pg_decr_and_check(
+ _barrier_pg_decrement(
bad_result, execution_id="exec-U", callback_descriptor=_CALLBACK
)
- def test_registered_under_canonical_name(self):
- assert barrier_pg_decr_and_check.name == "barrier_pg_decr_and_check"
-
-
class TestDecrementCoreExtraction:
"""The decrement logic lives in a plain ``_barrier_pg_decrement`` core so the
9e PR 2c PG-consumed path can call it in-body (a PG-consumed task fires no
@@ -1149,7 +1192,7 @@ class TestDecrementCoreExtraction:
# forwarding test below, this would catch a renamed/reordered core param
# — the core is NOT mocked away.
_seed(barrier_db, "exec-wrap", 3)
- out = barrier_pg_decr_and_check(
+ out = _barrier_pg_decrement(
{"f": 1}, execution_id="exec-wrap", callback_descriptor=_CALLBACK
)
assert out["status"] == "pending"
@@ -1157,21 +1200,9 @@ class TestDecrementCoreExtraction:
assert remaining == 2
assert results == [{"f": 1}]
- def test_worker_task_forwards_kwargs_verbatim(self):
- # Complements the real-row test above by pinning the keyword-forwarding
- # contract explicitly: the wrapper passes result + execution_id +
- # callback_descriptor straight through, returning the core's result.
- sentinel = {"status": "pending", "remaining": 9}
- with patch.object(
- pg_barrier, "_barrier_pg_decrement", return_value=sentinel
- ) as core:
- out = barrier_pg_decr_and_check(
- {"f": 1}, execution_id="exec-d", callback_descriptor=_CALLBACK
- )
- assert out is sentinel
- core.assert_called_once_with(
- {"f": 1}, execution_id="exec-d", callback_descriptor=_CALLBACK
- )
+ # ``test_worker_task_forwards_kwargs_verbatim`` is gone with the
+ # ``barrier_pg_decr_and_check`` @worker_task wrapper it pinned: there is no
+ # wrapper left to forward anything, only the core called directly in-body.
def test_open_transaction_on_shared_conn_raises(self):
# The in-body contract is enforced loudly: a caller that enters with an
@@ -1210,8 +1241,9 @@ class TestAbort:
assert barrier_pg_abort.name == "barrier_pg_abort"
def test_max_retries_zero(self):
- # A Celery retry would replay the decrement and corrupt the count.
- assert barrier_pg_decr_and_check.max_retries == 0
+ # A retry would replay the teardown. (The decrement's no-replay contract
+ # is enforced by its in-body caller, not a task option — its @worker_task
+ # wrapper went with the Celery .link path.)
assert barrier_pg_abort.max_retries == 0
@@ -1236,15 +1268,15 @@ class TestDecrementAtomicityThroughTask:
conns.append(conn)
pg_barrier._local.conn = conn # this thread's own connection
try:
- out = barrier_pg_decr_and_check(
+ out = _barrier_pg_decrement(
{"t": label}, execution_id="exec-Z", callback_descriptor=_CALLBACK
)
statuses[label] = out["status"]
finally:
pg_barrier._local.conn = None
- with patch("celery.current_app.signature") as sig:
- sig.return_value.apply_async.return_value = MagicMock(id="cb")
+ with patch("queue_backend.dispatch.dispatch") as sig:
+ sig.return_value = MagicMock(id="cb")
threads = [threading.Thread(target=run, args=(x,)) for x in ("a", "b")]
for t in threads:
t.start()
@@ -1252,7 +1284,7 @@ class TestDecrementAtomicityThroughTask:
t.join(timeout=10)
assert all(not t.is_alive() for t in threads)
assert sorted(statuses.values()) == ["complete", "pending"]
- assert sig.return_value.apply_async.call_count == 1 # single fire
+ assert sig.call_count == 1 # single fire
for c in conns:
c.close()
@@ -1292,7 +1324,6 @@ class TestPgFireAndForgetMode:
callback_kwargs={"execution_id": "exec-pg"},
callback_queue="general",
app_instance=None,
- transport="pg_queue",
)
assert mock_dispatch.call_count == 2 # one per header, no .link
for i, call in enumerate(mock_dispatch.call_args_list):
@@ -1302,7 +1333,6 @@ class TestPgFireAndForgetMode:
ctx = call.kwargs["kwargs"]["_barrier_context"]
assert ctx["execution_id"] == "exec-pg"
assert ctx["batch_index"] == i
- assert ctx["callback_descriptor"]["transport"] == "pg_queue"
# The pre-existing kwarg on h0 survives alongside the injected context.
assert mock_dispatch.call_args_list[0].kwargs["kwargs"]["pre_existing"] == "keep"
@@ -1321,7 +1351,6 @@ class TestPgFireAndForgetMode:
callback_kwargs={"execution_id": "exec-reuse"},
callback_queue="general",
app_instance=None,
- transport="pg_queue",
)
with barrier_db.cursor() as cur:
cur.execute(
@@ -1432,7 +1461,7 @@ class TestPgFireAndForgetMode:
assert remaining == 2
def test_fire_barrier_callback_pg_self_chains_via_dispatch(self):
- descriptor = {**_CALLBACK, "transport": "pg_queue"}
+ descriptor = dict(_CALLBACK)
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
mock_dispatch.return_value = MagicMock(id="pg-cb-1")
cb_id = _fire_barrier_callback(descriptor, [{"r": 1}])
@@ -1444,7 +1473,7 @@ class TestPgFireAndForgetMode:
# The PG callback carries the _pg_transport marker so the aggregating
# callback can gate its at-least-once duplicate guard on it. The shared
# descriptor must NOT be mutated (a copy is dispatched).
- descriptor = {**_CALLBACK, "transport": "pg_queue"}
+ descriptor = dict(_CALLBACK)
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
mock_dispatch.return_value = MagicMock(id="pg-cb")
_fire_barrier_callback(descriptor, [{"r": 1}])
@@ -1452,21 +1481,11 @@ class TestPgFireAndForgetMode:
assert dispatched.get(pg_barrier.PG_TRANSPORT_CALLBACK_KWARG) is True
assert pg_barrier.PG_TRANSPORT_CALLBACK_KWARG not in descriptor["kwargs"]
- def test_fire_barrier_callback_legacy_omits_transport_marker(self):
- # The Celery .link path must NOT inject the marker → the callback's PG
- # guard stays a no-op on Celery (no redelivery to guard).
- with patch("celery.current_app.signature") as sig:
- sig.return_value.apply_async.return_value = MagicMock(id="celery-cb")
- _fire_barrier_callback(_CALLBACK, [{"r": 1}])
- passed = sig.call_args.kwargs["kwargs"]
- assert pg_barrier.PG_TRANSPORT_CALLBACK_KWARG not in passed
-
def test_fire_barrier_callback_pg_carries_fairness(self):
# greptile: the PG callback must ride the producer's org/priority (parity
# with the Celery path), reconstructed from the stored fairness_headers.
descriptor = {
**_CALLBACK,
- "transport": "pg_queue",
"fairness_headers": {
FAIRNESS_HEADER_NAME: {
"org_id": "org-9",
@@ -1485,20 +1504,12 @@ class TestPgFireAndForgetMode:
def test_fire_barrier_callback_pg_without_fairness_passes_none(self):
# No producer key → None (dispatch writes neutral defaults), not a crash.
- descriptor = {**_CALLBACK, "transport": "pg_queue"} # fairness_headers None
+ descriptor = dict(_CALLBACK) # fairness_headers None
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
mock_dispatch.return_value = MagicMock(id="pg-cb")
_fire_barrier_callback(descriptor, [{"r": 1}])
assert mock_dispatch.call_args.kwargs["fairness"] is None
- def test_fire_barrier_callback_legacy_uses_celery(self):
- # No backend marker → the .link-mode Celery dispatch (unchanged).
- with patch("celery.current_app.signature") as sig:
- sig.return_value.apply_async.return_value = MagicMock(id="celery-cb")
- cb_id = _fire_barrier_callback(_CALLBACK, [{"r": 1}])
- assert cb_id == "celery-cb"
- sig.assert_called_once()
-
def test_abort_clears_dedup_markers(self, barrier_db):
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
@@ -1542,7 +1553,6 @@ class TestPgFireAndForgetMode:
"kwargs": {"execution_id": "exec-last"},
"queue": "api_file_processing_callback",
"fairness_headers": None,
- "transport": "pg_queue",
}
ctx = {
"execution_id": "exec-last",
@@ -1621,7 +1631,6 @@ class TestPgFireAndForgetMode:
callback_kwargs={"execution_id": "exec-midfail"},
callback_queue="general",
app_instance=None,
- transport="pg_queue",
)
assert _row(barrier_db, "exec-midfail") is None # row deleted on failure
with barrier_db.cursor() as cur:
diff --git a/workers/tests/test_pg_callback_duplicate_guard.py b/workers/tests/test_pg_callback_duplicate_guard.py
index 656067959..d7d7e4d67 100644
--- a/workers/tests/test_pg_callback_duplicate_guard.py
+++ b/workers/tests/test_pg_callback_duplicate_guard.py
@@ -41,8 +41,9 @@ class TestCallbackAlreadyRan:
class TestCoreCallbackGuard:
- """``_process_batch_callback_core`` short-circuits a PG duplicate before any
- side effect (using the status the context already carries), and only on PG.
+ """``_process_batch_callback_core`` short-circuits a duplicate before any side
+ effect, using the status the context already carries. Unconditional since
+ UN-4078 — the wire marker is still popped, but nothing branches on it.
"""
def _context(self, status):
@@ -88,16 +89,22 @@ class TestCoreCallbackGuard:
with pytest.raises(AssertionError, match="side effects must not run"):
self._run(is_pg=True, status=ExecutionStatus.ERROR.value)
- def test_celery_never_skips(self):
- # No marker → guard skipped entirely; the callback proceeds even on a
- # COMPLETED status — proving zero Celery regression.
- with pytest.raises(AssertionError, match="side effects must not run"):
- self._run(is_pg=False, status=ExecutionStatus.COMPLETED.value)
+ def test_guard_holds_without_the_marker(self):
+ # UN-4078: the guard no longer keys on the wire marker — PG is the only
+ # transport, so a COMPLETED redelivery is skipped whether or not the
+ # producer stamped it (an old producer that omits it must not re-fire
+ # webhooks / billing).
+ out, _extract, determine = self._run(
+ is_pg=False, status=ExecutionStatus.COMPLETED.value
+ )
+ assert out["duplicate_callback_skipped"] is True
+ determine.assert_not_called()
class TestApiCallbackGuard:
- """``process_batch_callback_api`` short-circuits a PG duplicate using the
- execution status it already fetches (no extra round-trip).
+ """``process_batch_callback_api`` short-circuits a duplicate using the
+ execution status it already fetches (no extra round-trip). Unconditional
+ since UN-4078.
"""
def _run(self, *, is_pg: bool, status: str):
@@ -131,6 +138,8 @@ class TestApiCallbackGuard:
with pytest.raises(AssertionError, match="side effects must not run"):
self._run(is_pg=True, status=ExecutionStatus.EXECUTING.value)
- def test_celery_never_skips(self):
- with pytest.raises(AssertionError, match="side effects must not run"):
- self._run(is_pg=False, status=ExecutionStatus.COMPLETED.value)
+ def test_guard_holds_without_the_marker(self):
+ # UN-4078: see the core-callback twin — the guard is unconditional now.
+ out, determine = self._run(is_pg=False, status=ExecutionStatus.COMPLETED.value)
+ assert out["status"] == "skipped_duplicate_callback"
+ determine.assert_not_called()
diff --git a/workers/tests/test_pg_duplicate_write_guard.py b/workers/tests/test_pg_duplicate_write_guard.py
index a91c62553..619b0bae5 100644
--- a/workers/tests/test_pg_duplicate_write_guard.py
+++ b/workers/tests/test_pg_duplicate_write_guard.py
@@ -1,12 +1,11 @@
-"""PG-only duplicate-destination-write guard + active-file TTL cap.
+"""Duplicate-destination-write guard + active-file TTL cap.
-On the PG (at-least-once) transport a batch can be re-run after a crash /
+The queue is at-least-once, so a batch can be re-run after a crash /
reaper-recovery, and such a re-run bypasses discovery's FileHistory filter. The
destination path therefore re-checks FileHistory by hash right before the write
and skips it when the file already completed in a prior run — closing the
-duplicate-write window. The guard is a no-op on the Celery transport (that path
-is unchanged), honours use_file_history (ETL "rewrite every run"), and is
-fail-open on any lookup error (never blocks a real write).
+duplicate-write window. The guard honours use_file_history (ETL "rewrite every
+run") and is fail-open on any lookup error (never blocks a real write).
"""
from __future__ import annotations
@@ -17,7 +16,6 @@ import pytest
from unstract.core.data_models import (
ExecutionStatus,
FileHashData,
- WorkflowTransport,
)
from shared.models.file_processing import FileProcessingContext
@@ -27,8 +25,6 @@ from shared.workflow.execution.active_file_manager import (
)
from shared.workflow.execution.service import WorkerWorkflowExecutionService
-PG = WorkflowTransport.PG_QUEUE.value
-CELERY = WorkflowTransport.CELERY.value
def _svc(api_client=None):
@@ -52,33 +48,26 @@ def _history(found=True, status=ExecutionStatus.COMPLETED.value):
return {"found": found, "file_history": {"status": status} if found else None}
-def _call(svc, *, transport, use_file_history=True, is_api=False, file_hash=None):
- return svc._pg_destination_already_written(
+def _call(svc, *, use_file_history=True, is_api=False, file_hash=None):
+ return svc._destination_already_written(
file_hash=file_hash or _file_hash(),
- transport=transport,
use_file_history=use_file_history,
workflow_id="wf-1",
is_api=is_api,
)
-def test_celery_transport_never_looks_up_or_skips():
- svc = _svc()
- assert _call(svc, transport=CELERY) is False
- svc.api_client.get_file_history_by_cache_key.assert_not_called()
-
-
def test_use_file_history_false_never_looks_up_or_skips():
# ETL/TASK "rewrite every run" contract — never skip its write.
svc = _svc()
- assert _call(svc, transport=PG, use_file_history=False) is False
+ assert _call(svc, use_file_history=False) is False
svc.api_client.get_file_history_by_cache_key.assert_not_called()
def test_pg_skips_when_completed_history_exists():
svc = _svc()
svc.api_client.get_file_history_by_cache_key.return_value = _history()
- assert _call(svc, transport=PG) is True
+ assert _call(svc) is True
@pytest.mark.parametrize(
@@ -92,7 +81,7 @@ def test_pg_skips_when_completed_history_exists():
def test_pg_proceeds_when_not_completed(history):
svc = _svc()
svc.api_client.get_file_history_by_cache_key.return_value = history
- assert _call(svc, transport=PG) is False
+ assert _call(svc) is False
def test_pg_fails_open_and_logs_on_lookup_error():
@@ -101,40 +90,33 @@ def test_pg_fails_open_and_logs_on_lookup_error():
with mock.patch(
"shared.workflow.execution.service.logger"
) as log:
- assert _call(svc, transport=PG) is False # fail-open
+ assert _call(svc) is False # fail-open
log.error.assert_called_once() # loud (Sentry-visible), not a silent warning
def test_pg_proceeds_when_no_cache_key():
svc = _svc()
- assert _call(svc, transport=PG, file_hash=_file_hash(h=None)) is False
+ assert _call(svc, file_hash=_file_hash(h=None)) is False
svc.api_client.get_file_history_by_cache_key.assert_not_called()
def test_api_execution_matches_on_hash_only():
svc = _svc()
svc.api_client.get_file_history_by_cache_key.return_value = _history()
- _call(svc, transport=PG, is_api=True)
+ _call(svc, is_api=True)
_, kwargs = svc.api_client.get_file_history_by_cache_key.call_args
assert kwargs["file_path"] is None
-def test_file_processing_context_carries_transport():
- ctx = FileProcessingContext(
- file_data=mock.Mock(),
- file_hash=_file_hash(),
- api_client=mock.Mock(),
- workflow_execution={},
- transport=PG,
- )
- assert ctx.transport == PG
-
+def test_handle_destination_processing_consults_guard_and_skips_on_hit():
+ """Real-call-site wiring: `_handle_destination_processing` consults the guard
+ and, on a hit, must NOT call `destination.handle_output`.
-def test_handle_destination_processing_forwards_transport_and_skips_on_hit():
- """Real-call-site wiring: `_handle_destination_processing` forwards the
- context's transport (not a non-existent `file_data.transport`) to the guard,
- and on a hit it must NOT call `destination.handle_output`. This is exactly
- the regression the original dead `getattr(file_data,'transport')` introduced.
+ This pins the skip, which is the behaviour that closes the duplicate-write
+ window. It previously also pinned that the transport reached the guard from
+ the context rather than a non-existent `file_data.transport` — that argument
+ is gone with the transport itself (UN-4078), so the guard now applies
+ unconditionally and there is no threading left to get wrong.
"""
svc = _svc()
file_hash = _file_hash()
@@ -144,7 +126,6 @@ def test_handle_destination_processing_forwards_transport_and_skips_on_hit():
file_hash=file_hash,
api_client=svc.api_client,
workflow_execution={},
- transport=PG,
)
workflow = mock.Mock()
workflow.destination_config.to_dict.return_value = {}
@@ -156,7 +137,7 @@ def test_handle_destination_processing_forwards_transport_and_skips_on_hit():
"shared.workflow.execution.service.WorkerDestinationConnector"
) as WDC,
mock.patch.object(
- svc, "_pg_destination_already_written", return_value=True
+ svc, "_destination_already_written", return_value=True
) as guard,
):
WDC.from_config.return_value = destination
@@ -172,7 +153,6 @@ def test_handle_destination_processing_forwards_transport_and_skips_on_hit():
is_api=False,
)
- assert guard.call_args.kwargs["transport"] == PG # threaded from the context
assert guard.call_args.kwargs["use_file_history"] is True
destination.handle_output.assert_not_called() # skipped, no duplicate write
assert result.processed is False
diff --git a/workers/tests/test_pg_finalization_fixes.py b/workers/tests/test_pg_finalization_fixes.py
index b1daad5c2..04b96464e 100644
--- a/workers/tests/test_pg_finalization_fixes.py
+++ b/workers/tests/test_pg_finalization_fixes.py
@@ -1,7 +1,9 @@
"""Unit tests for the PG finalization-strand fixes.
-- L2: ``_update_execution_status_unified`` re-raises on the PG path (so the failed
- finalization vt-redelivers / poison-drops) but swallows on the Celery path.
+- L2: ``_update_execution_status_unified`` re-raises a failed write by default (so
+ the failed finalization vt-redelivers / poison-drops). Since UN-4078 that is
+ unconditional — the Celery swallow is gone — and ``raise_on_failure=False`` is
+ reserved for the one caller already handling a failure.
- L3: ``_backoff_with_jitter`` spreads retries across [base/2, base] (equal jitter)
so concurrent workers don't retry in synchronized waves. Unconditional since
UN-4046 — it was gated on ``pg_queue_enabled`` to keep the Celery flow on the
@@ -44,10 +46,10 @@ class TestBackoffJitter:
assert len({_backoff_with_jitter(1.0, 3) for _ in range(64)}) > 1
-class TestPGFinalizationReraise:
+class TestFinalizationReraise:
AGG = {"total_files": 1, "successful_files": 1, "failed_files": 0}
- def _run(self, is_pg: bool, fail: bool):
+ def _run(self, raise_on_failure: bool, fail: bool):
from callback.tasks import _update_execution_status_unified
api = MagicMock()
@@ -61,25 +63,26 @@ class TestPGFinalizationReraise:
final_status="COMPLETED",
aggregated_results=self.AGG,
organization_id="org-1",
- is_pg=is_pg,
+ raise_on_failure=raise_on_failure,
),
)
- def test_pg_reraises_on_write_failure(self):
- # The whole point: a failed finalization on PG must propagate so the
- # message vt-redelivers instead of stranding the execution.
- _api, call = self._run(is_pg=True, fail=True)
+ def test_reraises_on_write_failure(self):
+ # The whole point: a failed finalization must propagate so the message
+ # vt-redelivers instead of stranding the execution.
+ _api, call = self._run(raise_on_failure=True, fail=True)
with pytest.raises(RuntimeError):
call()
- def test_celery_swallows_on_write_failure(self):
- # Celery behavior is unchanged — swallow and return the error dict.
- _api, call = self._run(is_pg=False, fail=True)
+ def test_opt_out_swallows_on_write_failure(self):
+ # raise_on_failure=False: the ERROR-marking write inside an except block,
+ # where a raise would mask the original failure.
+ _api, call = self._run(raise_on_failure=False, fail=True)
result = call()
assert result["status"] == "failed"
- def test_success_path_returns_completed_for_pg(self):
- api, call = self._run(is_pg=True, fail=False)
+ def test_success_path_returns_completed(self):
+ api, call = self._run(raise_on_failure=True, fail=False)
result = call()
assert result["status"] == "completed"
api.update_workflow_execution_status.assert_called_once()
diff --git a/workers/tests/test_pg_orchestration_failure.py b/workers/tests/test_pg_orchestration_failure.py
index 75adc03a1..5d77f74ce 100644
--- a/workers/tests/test_pg_orchestration_failure.py
+++ b/workers/tests/test_pg_orchestration_failure.py
@@ -6,9 +6,9 @@ counter reconciliation, and the guarantee that **neither a UI-logging hiccup nor
a status-update failure may disturb the caller's original exception/flow** (it
must never raise, so the handler's own re-raise / error-response always runs).
-The Celery path is byte-identical by construction: this recorder is only invoked
-on the ``is_pg_transport`` branch, so a Celery failure never reaches it (and so
-never sends file aggregates).
+It used to be reached only on the ``is_pg_transport`` branch, with the Celery
+path keeping a bare status update; with the transport gone (UN-4078) it is the
+single failure-recording path for both workers.
"""
from __future__ import annotations
diff --git a/workers/tests/test_plugin_loader.py b/workers/tests/test_plugin_loader.py
index 9e682a8fb..4ac720cd9 100644
--- a/workers/tests/test_plugin_loader.py
+++ b/workers/tests/test_plugin_loader.py
@@ -7,7 +7,7 @@ Verifies:
4. ExecutorPluginLoader.get() discovers entry-point-based plugins (mocked)
5. ExecutorPluginLoader.discover_executors() loads cloud executors (mocked)
6. text_processor.add_hex_line_numbers()
-7. ExecutionDispatcher._get_queue() naming convention
+7. Queue-per-executor naming convention (QUEUE_PREFIX)
8. Protocol classes importable and runtime-checkable
9. executors/__init__.py triggers discover_executors()
"""
@@ -17,7 +17,7 @@ from unittest.mock import MagicMock, patch
import pytest
from executor.executors.plugins.loader import ExecutorPluginLoader
from executor.executors.plugins.text_processor import add_hex_line_numbers
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
+from unstract.workflow_execution.executor_rpc import QUEUE_PREFIX
@pytest.fixture(autouse=True)
@@ -232,35 +232,31 @@ class TestTextProcessor:
class TestQueuePerExecutor:
- def test_get_queue_legacy(self):
- assert ExecutionDispatcher._get_queue("legacy") == "celery_executor_legacy"
+ """Queue-per-executor naming, now owned solely by the PG executor RPC.
- def test_get_queue_table(self):
- assert ExecutionDispatcher._get_queue("table") == "celery_executor_table"
-
- def test_get_queue_smart_table(self):
- assert (
- ExecutionDispatcher._get_queue("smart_table") == "celery_executor_smart_table"
- )
-
- def test_get_queue_simple_prompt_studio(self):
- assert (
- ExecutionDispatcher._get_queue("simple_prompt_studio")
- == "celery_executor_simple_prompt_studio"
- )
+ The names are ``celery_executor_*`` for historical reasons and are NOT dead
+ Celery surface: ``worker-pg-executor`` subscribes to exactly these strings
+ (chart ``workerPgExecutor.env.WORKER_PG_QUEUE_CONSUMER_QUEUE``). Renaming the
+ prefix without changing the chart would silently strand every tool execution,
+ so these assertions pin the literal wire names rather than deriving them.
+ """
- def test_get_queue_agentic(self):
- assert ExecutionDispatcher._get_queue("agentic") == "celery_executor_agentic"
+ @pytest.mark.parametrize(
+ "executor_name",
+ ["legacy", "table", "smart_table", "simple_prompt_studio", "agentic"],
+ )
+ def test_queue_for_known_executor(self, executor_name):
+ assert f"{QUEUE_PREFIX}{executor_name}" == f"celery_executor_{executor_name}"
- def test_get_queue_arbitrary_name(self):
+ def test_queue_for_arbitrary_name(self):
"""Any executor_name works — no whitelist."""
- assert ExecutionDispatcher._get_queue("my_custom") == "celery_executor_my_custom"
+ assert f"{QUEUE_PREFIX}my_custom" == "celery_executor_my_custom"
- def test_queue_name_enum_matches_dispatcher(self):
- """QueueName.EXECUTOR matches what dispatcher generates for 'legacy'."""
+ def test_queue_name_enum_matches_prefix(self):
+ """QueueName.EXECUTOR matches what the PG dispatcher builds for 'legacy'."""
from shared.enums.worker_enums import QueueName
- assert QueueName.EXECUTOR.value == ExecutionDispatcher._get_queue("legacy")
+ assert QueueName.EXECUTOR.value == f"{QUEUE_PREFIX}legacy"
# ── 6. Protocol classes importable ──────────────────────────────────
diff --git a/workers/tests/test_plugin_migration_regression.py b/workers/tests/test_plugin_migration_regression.py
index 29d4d81e6..b2a5046b5 100644
--- a/workers/tests/test_plugin_migration_regression.py
+++ b/workers/tests/test_plugin_migration_regression.py
@@ -8,11 +8,13 @@ from unittest.mock import MagicMock, patch
import pytest
from unstract.sdk1.execution.context import ExecutionContext, Operation
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.sdk1.execution.executor import BaseExecutor
from unstract.sdk1.execution.orchestrator import ExecutionOrchestrator
from unstract.sdk1.execution.registry import ExecutorRegistry
from unstract.sdk1.execution.result import ExecutionResult
+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher
+
+from .executor_dispatch_fakes import FakeExecutorTransport, queue_for
# ---------------------------------------------------------------------------
# Fixtures
@@ -436,15 +438,13 @@ class TestPluginDiscoveryToDispatchFlow:
# Step 2: Verify registration
assert "discovered" in ExecutorRegistry.list_executors()
- # Step 3: Dispatch via mock Celery
- mock_app = MagicMock()
- mock_result = MagicMock()
- mock_result.get.return_value = ExecutionResult(
- success=True, data={"output": "discovered_result"}
- ).to_dict()
- mock_app.send_task.return_value = mock_result
-
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
+ # Step 3: Dispatch via a fake executor transport
+ transport = FakeExecutorTransport(
+ result=ExecutionResult(
+ success=True, data={"output": "discovered_result"}
+ ).to_dict()
+ )
+ dispatcher = PgExecutionDispatcher(transport)
ctx = ExecutionContext(
executor_name="discovered",
operation="custom_op",
@@ -458,8 +458,7 @@ class TestPluginDiscoveryToDispatchFlow:
assert result.data["output"] == "discovered_result"
# Step 5: Verify queue routing
- call_kwargs = mock_app.send_task.call_args
- assert call_kwargs.kwargs["queue"] == "celery_executor_discovered"
+ assert transport.queue == "celery_executor_discovered"
# ---------------------------------------------------------------------------
@@ -481,7 +480,7 @@ class TestQueueRoutingAllExecutors:
list(EXECUTOR_QUEUE_MAP.items()),
)
def test_queue_name_for_executor(self, executor_name, expected_queue):
- assert ExecutionDispatcher._get_queue(executor_name) == expected_queue
+ assert queue_for(executor_name) == expected_queue
# ---------------------------------------------------------------------------
diff --git a/workers/tests/test_prompt_studio_dispatch.py b/workers/tests/test_prompt_studio_dispatch.py
index 0f21605e9..9798215ca 100644
--- a/workers/tests/test_prompt_studio_dispatch.py
+++ b/workers/tests/test_prompt_studio_dispatch.py
@@ -1,16 +1,16 @@
"""IDE path integration tests through executor chain.
-Phase 4 replaces PromptTool HTTP calls in PromptStudioHelper with
-ExecutionDispatcher → executor worker → LegacyExecutor.
+Phase 4 replaced PromptTool HTTP calls in PromptStudioHelper with a dispatch to
+the executor worker → LegacyExecutor.
-These tests build the EXACT payloads that prompt_studio_helper.py
-now sends via ExecutionDispatcher, push them through the full Celery
-eager-mode chain, and verify the results match what the IDE expects.
+These tests build the EXACT payloads that prompt_studio_helper.py now sends,
+push them through the full chain with the executor task running in-process, and
+verify the results match what the IDE expects.
This validates the full contract:
prompt_studio_helper builds payload
→ ExecutionContext(execution_source="ide", ...)
- → Celery task → LegacyExecutor._handle_X()
+ → execute_extraction task → LegacyExecutor._handle_X()
→ ExecutionResult → result.data used by IDE
All tests use execution_source="ide" to match the real IDE path.
@@ -23,9 +23,11 @@ from executor.executors.constants import (
PromptServiceConstants as PSKeys,
)
from unstract.sdk1.execution.context import ExecutionContext
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.sdk1.execution.registry import ExecutorRegistry
from unstract.sdk1.execution.result import ExecutionResult
+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher
+
+from .executor_dispatch_fakes import EagerExecutorTransport
# ---------------------------------------------------------------------------
# Patch targets (same as Phase 2 sanity)
@@ -654,41 +656,28 @@ class TestIDESinglePass:
class TestIDEDispatcherIntegration:
- """Test ExecutionDispatcher dispatch() with IDE payloads in eager mode.
+ """Dispatcher round trips with IDE payloads, running the task in-process.
- Celery's send_task() doesn't work with eager mode for AsyncResult.get(),
- so we patch send_task to delegate to task.apply() instead.
+ ``EagerExecutorTransport`` runs ``execute_extraction`` synchronously instead
+ of enqueueing it, so each test still covers dispatcher → task → executor →
+ result. This replaces the old ``send_task``-patching dance, which existed
+ only because Celery's eager mode does not support ``AsyncResult.get()``.
"""
- @staticmethod
- def _patch_send_task(eager_app):
- """Patch send_task on eager_app to use task.apply()."""
- original_send_task = eager_app.send_task
-
- def patched_send_task(name, args=None, kwargs=None, **opts):
- task = eager_app.tasks[name]
- return task.apply(args=args, kwargs=kwargs)
-
- eager_app.send_task = patched_send_task
- return original_send_task
-
@patch(_PATCH_FS)
@patch(_PATCH_X2TEXT)
def test_dispatcher_extract_round_trip(self, mock_x2text_cls, mock_get_fs, eager_app):
- """ExecutionDispatcher.dispatch() → extract → ExecutionResult."""
+ """Dispatcher round trip → extract → ExecutionResult."""
mock_x2text = MagicMock()
mock_x2text.process.return_value = _mock_process_response("dispatcher extracted")
mock_x2text.x2text_instance = MagicMock()
mock_x2text_cls.return_value = mock_x2text
mock_get_fs.return_value = MagicMock()
- original = self._patch_send_task(eager_app)
- try:
- dispatcher = ExecutionDispatcher(celery_app=eager_app)
- ctx = _ide_extract_ctx()
- result = dispatcher.dispatch(ctx)
- finally:
- eager_app.send_task = original
+ dispatcher = PgExecutionDispatcher(
+ EagerExecutorTransport(eager_app.tasks["execute_extraction"])
+ )
+ result = dispatcher.dispatch(_ide_extract_ctx())
assert result.success is True
assert result.data["extracted_text"] == "dispatcher extracted"
@@ -700,18 +689,15 @@ class TestIDEDispatcherIntegration:
def test_dispatcher_answer_prompt_round_trip(
self, mock_shim_cls, mock_deps, _mock_idx, _mock_plugin, eager_app
):
- """ExecutionDispatcher.dispatch() → answer_prompt → ExecutionResult."""
+ """Dispatcher round trip → answer_prompt → ExecutionResult."""
llm = _mock_llm("dispatcher answer")
mock_deps.return_value = _mock_prompt_deps(llm)
mock_shim_cls.return_value = MagicMock()
- original = self._patch_send_task(eager_app)
- try:
- dispatcher = ExecutionDispatcher(celery_app=eager_app)
- ctx = _ide_answer_prompt_ctx()
- result = dispatcher.dispatch(ctx)
- finally:
- eager_app.send_task = original
+ dispatcher = PgExecutionDispatcher(
+ EagerExecutorTransport(eager_app.tasks["execute_extraction"])
+ )
+ result = dispatcher.dispatch(_ide_answer_prompt_ctx())
assert result.success is True
assert result.data["output"]["invoice_number"] == "dispatcher answer"
@@ -724,18 +710,15 @@ class TestIDEDispatcherIntegration:
def test_dispatcher_single_pass_round_trip(
self, mock_shim_cls, mock_deps, _mock_idx, _mock_plugin, eager_app
):
- """ExecutionDispatcher.dispatch() → single_pass → ExecutionResult."""
+ """Dispatcher round trip → single_pass → ExecutionResult."""
llm = _mock_llm("sp dispatch")
mock_deps.return_value = _mock_prompt_deps(llm)
mock_shim_cls.return_value = MagicMock()
- original = self._patch_send_task(eager_app)
- try:
- dispatcher = ExecutionDispatcher(celery_app=eager_app)
- ctx = _ide_single_pass_ctx()
- result = dispatcher.dispatch(ctx)
- finally:
- eager_app.send_task = original
+ dispatcher = PgExecutionDispatcher(
+ EagerExecutorTransport(eager_app.tasks["execute_extraction"])
+ )
+ result = dispatcher.dispatch(_ide_single_pass_ctx())
assert result.success is True
assert "revenue" in result.data["output"]
@@ -743,7 +726,7 @@ class TestIDEDispatcherIntegration:
@patch(_PATCH_FS)
@patch(_PATCH_INDEX_DEPS)
def test_dispatcher_index_round_trip(self, mock_deps, mock_get_fs, eager_app):
- """ExecutionDispatcher.dispatch() → index → ExecutionResult."""
+ """Dispatcher round trip → index → ExecutionResult."""
mock_index_cls = MagicMock()
mock_index = MagicMock()
mock_index.generate_index_key.return_value = "doc-dispatch-idx"
@@ -754,13 +737,10 @@ class TestIDEDispatcherIntegration:
mock_deps.return_value = (mock_index_cls, MagicMock(), MagicMock())
mock_get_fs.return_value = MagicMock()
- original = self._patch_send_task(eager_app)
- try:
- dispatcher = ExecutionDispatcher(celery_app=eager_app)
- ctx = _ide_index_ctx()
- result = dispatcher.dispatch(ctx)
- finally:
- eager_app.send_task = original
+ dispatcher = PgExecutionDispatcher(
+ EagerExecutorTransport(eager_app.tasks["execute_extraction"])
+ )
+ result = dispatcher.dispatch(_ide_index_ctx())
assert result.success is True
assert result.data["doc_id"] == "doc-dispatch-idx"
diff --git a/workers/tests/test_queue_backend_seam.py b/workers/tests/test_queue_backend_seam.py
index 12e85f9fd..219e56009 100644
--- a/workers/tests/test_queue_backend_seam.py
+++ b/workers/tests/test_queue_backend_seam.py
@@ -183,32 +183,20 @@ class TestPublicSurface:
def test_all_exports(self):
import queue_backend
- # Phase 6a added Barrier / BarrierHandle / CeleryChordBarrier.
- # Phase 6b adds RedisDecrBarrier + barrier_decr_and_check
- # (registered as a Celery task on import) + the BarrierBackend
- # enum + the get_barrier factory that the WORKER_BARRIER_BACKEND
- # env flag drives.
- # Phase 8a adds QueueBackend + select_backend — the queue-transport
- # routing gate that the WORKER_PG_QUEUE_ENABLED_TASKS allow-list
- # drives.
- # The PG-queue barrier work adds the Postgres barrier surface:
- # PgBarrier + its barrier_pg_decr_and_check / barrier_pg_abort tasks,
- # selected when WORKER_BARRIER_BACKEND routes to the PG backend.
+ # The surface after UN-4078 removed the Celery transport. What went:
+ # CeleryChordBarrier / RedisDecrBarrier (the two other substrates),
+ # BarrierBackend + get_barrier (the WORKER_BARRIER_BACKEND factory),
+ # barrier_abort / barrier_decr_and_check (the Redis link tasks), and
+ # barrier_pg_decr_and_check (the PG link-task wrapper — the decrement
+ # now runs in-body, so only the abort is still a task).
assert set(queue_backend.__all__) == {
"Barrier",
- "BarrierBackend",
"BarrierHandle",
- "CeleryChordBarrier",
"FairnessKey",
"PgBarrier",
"QueueBackend",
- "RedisDecrBarrier",
- "barrier_abort",
- "barrier_decr_and_check",
"barrier_pg_abort",
- "barrier_pg_decr_and_check",
"dispatch",
- "get_barrier",
"select_backend",
"worker_task",
}
diff --git a/workers/tests/test_redis_barrier.py b/workers/tests/test_redis_barrier.py
deleted file mode 100644
index de098cf58..000000000
--- a/workers/tests/test_redis_barrier.py
+++ /dev/null
@@ -1,972 +0,0 @@
-"""Characterisation tests for ``RedisDecrBarrier`` (PG Queue Phase 6b).
-
-The risky behaviour-change PR replacing Celery's chord aggregation
-primitive with the labs-design ``DECR remaining`` + ``RPUSH results``
-pattern (see ``queue_backend/redis_barrier.py``).
-
-Three layers:
-
-1. **Protocol shape** — ``RedisDecrBarrier`` satisfies the ``Barrier``
- Protocol; the handle satisfies ``BarrierHandle``.
-2. **Wire model** — ``enqueue`` initialises the per-execution counter
- + results list with TTL, stamps fairness on each header task,
- attaches the ``barrier_decr_and_check`` link, and dispatches via
- ``apply_async``.
-3. **Link aggregation** — the ``barrier_decr_and_check`` task runs the
- atomic Lua script, fires the callback exactly when remaining reads
- 0, passes the aggregated results, and respects fairness on the
- callback signature.
-
-Redis is mocked via ``MagicMock`` — the contract under test is the
-sequence of Redis ops + their args, not Redis's own correctness.
-A future integration test (against a real Redis) would belong in
-``test_redis_barrier_integration.py``.
-"""
-
-from __future__ import annotations
-
-import json
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from queue_backend import (
- Barrier,
- BarrierHandle,
- RedisDecrBarrier,
- barrier_decr_and_check,
-)
-from queue_backend.fairness import FAIRNESS_HEADER_NAME, FairnessKey, WorkloadType
-from queue_backend.redis_barrier import (
- _KEY_TTL_DEFAULT_SECONDS,
- _remaining_key,
- _results_key,
-)
-
-
-# --- Protocol shape ---
-
-
-class TestRedisDecrBarrierProtocolShape:
- def test_satisfies_barrier_protocol(self):
- barrier: Barrier = RedisDecrBarrier()
- assert callable(getattr(barrier, "enqueue", None))
-
- def test_handle_satisfies_barrier_handle(self):
- from queue_backend.redis_barrier import _RedisBarrierHandle
-
- handle: BarrierHandle = _RedisBarrierHandle(id="exec-1")
- assert handle.id == "exec-1"
- assert isinstance(handle.id, str)
-
-
-# --- Enqueue: wire model ---
-
-
-def redis_mock():
- """Mock the Redis client used by ``RedisDecrBarrier``."""
- return MagicMock(name="redis_client")
-
-
-def barrier(redis_mock):
- """``RedisDecrBarrier`` with an injected mock client.
-
- Production builds its client from env via ``create_redis_client``;
- tests inject so no Redis is required.
- """
- return RedisDecrBarrier(redis_client=redis_mock)
-
-
-class TestRedisDecrBarrierEnqueue:
- def test_empty_header_returns_none_and_touches_no_redis(self, barrier, redis_mock):
- """Mirrors the ``CeleryChordBarrier`` zero-header contract:
- ``None`` is the *sole* signal of no-work-enqueued, and we
- don't pay the Redis round-trip for it."""
- result = barrier.enqueue(
- [],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
- assert result is None
- # No SET / DELETE on the empty path — short-circuit before any
- # Redis round-trip.
- redis_mock.set.assert_not_called()
- redis_mock.delete.assert_not_called()
-
- def test_missing_execution_id_raises(self, barrier):
- """``RedisDecrBarrier`` uses ``execution_id`` as the key suffix
- for ``remaining:{exec_id}`` / ``results:{exec_id}``. Without
- it we can't isolate this execution's counter — fail loudly
- instead of routing into a global namespace collision."""
- header = [MagicMock(name="h1")]
- with pytest.raises(ValueError, match=r"execution_id"):
- barrier.enqueue(
- header,
- callback_task_name="cb",
- # No execution_id.
- callback_kwargs={"pipeline_id": "p-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
-
- def test_initialises_remaining_counter_with_ttl(self, barrier, redis_mock):
- """``remaining:{exec_id}`` SET to ``len(header_tasks)`` with
- TTL — belt-and-suspenders cleanup if a link never fires for
- a task (e.g. worker crash mid-execution)."""
- header = [MagicMock(name="h1"), MagicMock(name="h2"), MagicMock(name="h3")]
- barrier.enqueue(
- header,
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-42"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
- redis_mock.set.assert_called_once_with(
- _remaining_key("exec-42"),
- 3,
- ex=_KEY_TTL_DEFAULT_SECONDS,
- )
- # ``results:{exec_id}`` AND ``abort_lock:{exec_id}`` are DEL'd
- # to clear any stale prior-run leftovers (e.g. from a retry
- # reusing the same execution_id). Without DELing the abort
- # lock here, a stale lock from a prior failed run would mask
- # a retry's failure as success.
- from queue_backend.redis_barrier import _abort_lock_key
-
- redis_mock.delete.assert_called_once_with(
- _results_key("exec-42"),
- _abort_lock_key("exec-42"),
- )
-
- def test_link_attached_to_each_header_task(self, barrier, redis_mock):
- """Every header task gets the ``barrier_decr_and_check`` link.
- Cloning (not mutating the caller's list) preserves the same
- guarantee ``CeleryChordBarrier`` makes for cross-tenant
- signature reuse."""
- h1 = MagicMock(name="h1")
- h2 = MagicMock(name="h2")
- barrier.enqueue(
- [h1, h2],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
- # Each task cloned, .link(...) called once on the clone,
- # apply_async called once on the clone.
- h1.clone.assert_called_once_with()
- h2.clone.assert_called_once_with()
- h1.clone.return_value.link.assert_called_once()
- h2.clone.return_value.link.assert_called_once()
- h1.clone.return_value.apply_async.assert_called_once_with()
- h2.clone.return_value.apply_async.assert_called_once_with()
- # The caller's signatures were NOT mutated (no .link on the
- # originals).
- h1.link.assert_not_called()
- h2.link.assert_not_called()
-
- def test_fairness_header_stamped_on_each_header_task(self, barrier, redis_mock):
- """``fairness.as_header()`` rides on every header signature as
- an additive AMQP header — same wire shape as
- ``CeleryChordBarrier``'s fairness plumbing."""
- h1 = MagicMock(name="h1")
- fairness = FairnessKey(org_id="org-x", workload_type=WorkloadType.API)
- barrier.enqueue(
- [h1],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- fairness=fairness,
- )
- # ``as_header()`` returns ``{FAIRNESS_HEADER_NAME: <dict>}``
- # (a nested dict, not a JSON string — see fairness.py:62).
- h1.clone.return_value.set.assert_called_once_with(
- headers=fairness.as_header()
- )
-
- def test_no_fairness_no_header_added(self, barrier, redis_mock):
- """When ``fairness=None``, no ``headers=`` kwarg on the
- signature — preserves wire equivalence for the
- non-fairness call path."""
- h1 = MagicMock(name="h1")
- barrier.enqueue(
- [h1],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
- h1.clone.return_value.set.assert_not_called()
-
- def test_returns_handle_with_execution_id(self, barrier, redis_mock):
- """``BarrierHandle.id`` is the execution id — what call sites
- log for chord-id tracing. (Under ``CeleryChordBarrier`` this
- was the chord aggregator's AsyncResult id; under
- ``RedisDecrBarrier`` we don't have such a task, so we expose
- the execution id which is a stable per-execution identifier
- that the existing log consumers don't depend on the
- underlying type of.)"""
- result = barrier.enqueue(
- [MagicMock()],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-42"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
- assert result is not None
- assert result.id == "exec-42"
- assert isinstance(result.id, str)
-
- def test_setup_failure_raises(self, barrier, redis_mock):
- """Any substrate failure (Redis down, network blip mid-setup)
- propagates as an exception — matches the ``Barrier`` Protocol:
- ``None`` is sole "no-op" signal, everything else raises."""
- redis_mock.set.side_effect = ConnectionError("redis is down")
- with pytest.raises(ConnectionError, match="redis is down"):
- barrier.enqueue(
- [MagicMock()],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
-
- def test_default_ttl_is_six_hours(self):
- """Lock in the chosen default. Accidental shrinkage to a few
- minutes would cause spurious callback fires on any execution
- longer than the new value; accidental expansion to days would
- leak orphaned keys. The default reflects the worst-case
- ``FILE_PROCESSING_TASK_TIME_LIMIT`` (3h) x 2x margin."""
- assert _KEY_TTL_DEFAULT_SECONDS == 6 * 60 * 60
-
- def test_ttl_overridable_via_env(self, barrier, redis_mock, monkeypatch):
- """Operators with substantially longer (or shorter) workflows
- can override the cleanup TTL via ``WORKER_BARRIER_KEY_TTL_SECONDS``.
-
- Set well above the default to confirm the env-driven path is
- actually read (not the constant)."""
- monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", str(3 * 24 * 3600))
- barrier.enqueue(
- [MagicMock()],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
- # ``ex=`` on the counter SET should be the overridden 3-day value.
- assert redis_mock.set.call_args.kwargs.get("ex") == 3 * 24 * 3600
-
- # Invalid-value handling tests live in ``TestTtlEnvValidation``
- # below — they now assert ``raises`` (matching ``get_barrier()``'s
- # loud-on-misconfig posture) rather than silent fallback.
-
-
-# --- Link task: aggregation ---
-
-
-class TestBarrierDecrAndCheckLink:
- """The ``barrier_decr_and_check`` worker task — the load-bearing
- aggregator. Runs after each header task and either decrements +
- aggregates (most calls) or fires the callback (the last call).
- """
-
- @patch("queue_backend.redis_barrier._get_redis_client")
- def test_pending_path_decrement_only(self, get_redis):
- """When the post-decrement counter reads >0, no callback
- dispatch — just ``RPUSH + DECR`` and return a pending status."""
- redis_client = MagicMock(name="redis")
- # Lua script returns [remaining, results-empty-list-since-pending].
- # ``register_script`` returns a callable that returns the script result.
- script = MagicMock(name="script", return_value=[2, []])
- redis_client.register_script.return_value = script
- get_redis.return_value = redis_client
-
- with patch("celery.current_app") as current_app:
- # Invoke the underlying function directly (bypass
- # Celery's @worker_task wrapper for unit testing).
- result = barrier_decr_and_check.run(
- {"batch_id": "b1", "status": "ok"},
- execution_id="exec-1",
- callback_descriptor={
- "task_name": "cb",
- "kwargs": {"execution_id": "exec-1"},
- "queue": "q",
- "fairness_headers": None,
- },
- )
-
- # Lua script invoked with the right keys + JSON-serialised
- # result + TTL seconds for the EXPIRE inside the script.
- script.assert_called_once()
- call_kwargs = script.call_args.kwargs
- assert call_kwargs["keys"] == [
- _remaining_key("exec-1"),
- _results_key("exec-1"),
- ]
- assert json.loads(call_kwargs["args"][0]) == {"batch_id": "b1", "status": "ok"}
- assert call_kwargs["args"][1] == _KEY_TTL_DEFAULT_SECONDS
-
- # No callback dispatch — remaining > 0.
- current_app.signature.assert_not_called()
- assert result == {"status": "pending", "remaining": 2}
-
- @patch("queue_backend.redis_barrier._get_redis_client")
- def test_complete_path_fires_callback_with_aggregated_results(self, get_redis):
- """When the post-decrement counter reads 0, the link reads the
- full results list (returned by the Lua script in the same
- atomic step), constructs the callback signature with the
- aggregated list as the first arg, and dispatches via
- ``apply_async``."""
- redis_client = MagicMock(name="redis")
- # Lua returns [0, [serialised result_1, result_2, result_3]].
- aggregated_raw = [
- json.dumps({"batch_id": "b1", "status": "ok"}),
- json.dumps({"batch_id": "b2", "status": "ok"}),
- json.dumps({"batch_id": "b3", "status": "ok"}),
- ]
- script = MagicMock(name="script", return_value=[0, aggregated_raw])
- redis_client.register_script.return_value = script
- get_redis.return_value = redis_client
-
- with patch("celery.current_app") as current_app:
- cb_sig = MagicMock(name="callback_signature")
- cb_sig.apply_async.return_value = MagicMock(id="callback-task-id-xyz")
- current_app.signature.return_value = cb_sig
-
- result = barrier_decr_and_check.run(
- {"batch_id": "b3", "status": "ok"},
- execution_id="exec-42",
- callback_descriptor={
- "task_name": "process_batch_callback_api",
- "kwargs": {
- "execution_id": "exec-42",
- "pipeline_id": "pipe-7",
- "organization_id": "org-x",
- },
- "queue": "api_file_processing_callback",
- "fairness_headers": None,
- },
- )
-
- # Callback signature constructed with aggregated results as
- # first arg, callback kwargs forwarded, queue pinned.
- current_app.signature.assert_called_once_with(
- "process_batch_callback_api",
- args=[
- [
- {"batch_id": "b1", "status": "ok"},
- {"batch_id": "b2", "status": "ok"},
- {"batch_id": "b3", "status": "ok"},
- ],
- ],
- kwargs={
- "execution_id": "exec-42",
- "pipeline_id": "pipe-7",
- "organization_id": "org-x",
- },
- queue="api_file_processing_callback",
- )
- cb_sig.apply_async.assert_called_once_with()
- # DELETE is now called from Python AFTER apply_async succeeds
- # (the Lua script no longer DELs in the ``== 0`` branch). This
- # is the load-bearing deferred-cleanup invariant: a dispatch
- # failure leaves the keys + TTL in place so the execution
- # isn't stranded.
- redis_client.delete.assert_called_once_with(
- _remaining_key("exec-42"),
- _results_key("exec-42"),
- )
-
- assert result["status"] == "complete"
- assert result["callback_task_id"] == "callback-task-id-xyz"
- assert result["aggregated_count"] == 3
-
- @patch("queue_backend.redis_barrier._get_redis_client")
- def test_complete_path_passes_fairness_header_to_callback(self, get_redis):
- """Fairness rides the callback signature too — same wire shape
- as ``CeleryChordBarrier``'s callback fairness plumbing."""
- redis_client = MagicMock(name="redis")
- script = MagicMock(name="script", return_value=[0, [json.dumps({"x": 1})]])
- redis_client.register_script.return_value = script
- get_redis.return_value = redis_client
-
- # Match the wire shape produced by ``FairnessKey.as_header()`` —
- # a nested dict, not a JSON string (see fairness.py:62).
- fairness_headers = {
- FAIRNESS_HEADER_NAME: {
- "org_id": "org-x",
- "workload_type": "non_api",
- "pipeline_priority": 5,
- }
- }
- with patch("celery.current_app") as current_app:
- current_app.signature.return_value = MagicMock(
- apply_async=MagicMock(return_value=MagicMock(id="cb-id"))
- )
-
- barrier_decr_and_check.run(
- {"x": 1},
- execution_id="exec-1",
- callback_descriptor={
- "task_name": "cb",
- "kwargs": {"execution_id": "exec-1"},
- "queue": "q",
- "fairness_headers": fairness_headers,
- },
- )
-
- # ``headers=fairness_headers`` MUST be on the signature kwargs.
- assert current_app.signature.call_args.kwargs.get("headers") == fairness_headers
-
- @patch("queue_backend.redis_barrier._get_redis_client")
- def test_unserialisable_result_raises_loudly(self, get_redis):
- """A header task returning something json.dumps can't handle
- is a typed-boundary regression. Raise loudly so the
- execution surfaces an error rather than silently dropping the
- result."""
- redis_client = MagicMock(name="redis")
- get_redis.return_value = redis_client
-
- # ``complex`` is a classic JSON-unserialisable leaf — no
- # ``__json__`` / ``__str__``-with-meaning, and ``default=str``
- # isn't applied (we dropped it). A regressed
- # ``BatchExecutionResult.to_dict()`` producing such a leaf
- # would land here. The test pins that the loud-failure
- # ``try/except`` actually surfaces the regression rather than
- # being a no-op.
- with pytest.raises((TypeError, ValueError)):
- barrier_decr_and_check.run(
- {"bad": complex(1, 2)},
- execution_id="exec-1",
- callback_descriptor={
- "task_name": "cb",
- "kwargs": {"execution_id": "exec-1"},
- "queue": "q",
- "fairness_headers": None,
- },
- )
-
- def test_link_task_is_registered_with_celery_under_canonical_name(self):
- """The link task must be importable + registered under
- ``barrier_decr_and_check`` — any worker that processes the
- link task's queue needs the name to resolve.
-
- Mixed-version rolling deploys depend on this: a worker
- running 6b-old code without the link task registered would
- fail with ``KeyError`` on the link's task name, which is a
- loud failure (no silent drop)."""
- assert barrier_decr_and_check.name == "barrier_decr_and_check"
-
- def test_link_max_retries_zero_prevents_double_decrement(self):
- """The link task MUST have ``max_retries=0``. The Lua script
- is atomic at the Redis level but the link itself is not
- idempotent: a Celery retry would replay the script and
- double-RPUSH + double-DECR, corrupting the aggregation
- (counter hits 0 prematurely; results list has duplicates).
- TTL-driven cleanup is the safety net for transient failures
- rather than Celery autoretry."""
- # Celery exposes ``max_retries`` on the task class.
- assert barrier_decr_and_check.max_retries == 0
-
- @patch("queue_backend.redis_barrier._get_redis_client")
- def test_negative_remaining_does_not_fire_callback(self, get_redis):
- """The Lua script returns ``remaining < 0`` when the counter
- was TTL-expired or already cleaned up. The Python side must
- NOT fire the callback in that branch — otherwise every
- subsequent task completion after a TTL expiry would dispatch
- a spurious callback."""
- redis_client = MagicMock(name="redis")
- # Lua returns ``[-1, []]`` — the ``< 0`` branch where keys
- # were already DEL'd (or never existed).
- script = MagicMock(name="script", return_value=[-1, []])
- redis_client.register_script.return_value = script
- get_redis.return_value = redis_client
-
- with patch("celery.current_app") as current_app:
- result = barrier_decr_and_check.run(
- {"x": 1},
- execution_id="exec-1",
- callback_descriptor={
- "task_name": "cb",
- "kwargs": {"execution_id": "exec-1"},
- "queue": "q",
- "fairness_headers": None,
- },
- )
-
- # No callback dispatch — the abandoned branch fires.
- current_app.signature.assert_not_called()
- assert result["status"] == "abandoned"
- assert result["remaining"] == -1
-
- @patch("queue_backend.redis_barrier._get_redis_client")
- def test_callback_dispatch_failure_preserves_keys_for_ttl(self, get_redis):
- """If ``apply_async`` raises after the Lua ``== 0`` branch
- returned, the barrier keys MUST remain in Redis (with their
- TTL still active) so the execution isn't stranded with no
- recovery path.
-
- Without deferred DEL, the previous implementation DEL'd both
- keys inside the Lua script before Python's ``apply_async``
- ran. A dispatch failure (broker outage, serialisation error,
- routing failure) then left the execution with: no keys, no
- TTL, no Celery retry (``max_retries=0``), no link_error
- fallback — strictly worse than the chord baseline.
-
- The fix: Lua's ``== 0`` branch returns the LRANGE results
- WITHOUT DELing. Python DELs both keys only after
- ``apply_async`` succeeds. A failure raises and the TTL
- eventually cleans up the keys."""
- redis_client = MagicMock(name="redis")
- # ``== 0`` branch — script returns one result; Lua no longer DELs.
- script = MagicMock(name="script", return_value=[0, [json.dumps({"x": 1})]])
- redis_client.register_script.return_value = script
- get_redis.return_value = redis_client
-
- with patch("celery.current_app") as current_app:
- cb_sig = MagicMock(name="callback_signature")
- # apply_async raises — broker outage, serialisation error, etc.
- cb_sig.apply_async.side_effect = RuntimeError("broker is down")
- current_app.signature.return_value = cb_sig
-
- with pytest.raises(RuntimeError, match=r"broker is down"):
- barrier_decr_and_check.run(
- {"x": 1},
- execution_id="exec-1",
- callback_descriptor={
- "task_name": "cb",
- "kwargs": {"execution_id": "exec-1"},
- "queue": "q",
- "fairness_headers": None,
- },
- )
-
- # DELETE MUST NOT have been called — the keys must remain in
- # Redis (with their existing TTL) so:
- # - Late-arriving link tasks (if any) cleanly hit the Lua
- # ``< 0`` branch and exit via the abandoned path.
- # - TTL eventually cleans the keys.
- # - The failure is visible via Celery's standard task-failure
- # channels rather than a silent infinite hang.
- redis_client.delete.assert_not_called()
-
-
-# --- TTL env validation ---
-
-
-class TestTtlEnvValidation:
- """Misconfigured ``WORKER_BARRIER_KEY_TTL_SECONDS`` must raise.
-
- Matches the posture of ``get_barrier()`` on a typo'd
- ``WORKER_BARRIER_BACKEND``: a misconfigured TTL shorter than
- execution wall-clock is a correctness issue (spurious callback
- fires per the Lua ``< 0`` branch), so operators get a loud
- on-startup signal rather than silent degradation.
- """
-
- def test_non_integer_raises(self, barrier, redis_mock, monkeypatch):
- monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "not-an-int")
- with pytest.raises(ValueError, match=r"not an integer"):
- barrier.enqueue(
- [MagicMock()],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
-
- def test_zero_raises(self, barrier, redis_mock, monkeypatch):
- monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "0")
- with pytest.raises(ValueError, match=r"positive integer"):
- barrier.enqueue(
- [MagicMock()],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
-
- def test_negative_raises(self, barrier, redis_mock, monkeypatch):
- monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "-1")
- with pytest.raises(ValueError, match=r"positive integer"):
- barrier.enqueue(
- [MagicMock()],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
-
-
-# --- Mid-loop dispatch failure ---
-
-
-class TestDispatchFailureCleanup:
- """If ``apply_async`` raises mid-fan-out, partially-dispatched
- tasks' links would otherwise corrupt the counter and the orphan
- state would leak. The barrier must DEL its keys before re-raising
- so the in-flight links' DECR returns negative → Lua ``< 0``
- branch cleanly abandons the execution.
- """
-
- def test_mid_loop_dispatch_failure_cleans_up_keys(
- self, barrier, redis_mock
- ):
- """Task K of N raises during ``apply_async`` — barrier must
- DEL both keys before propagating the exception."""
- h1 = MagicMock(name="h1")
- h2 = MagicMock(name="h2")
- h3_failing = MagicMock(name="h3_failing")
- h3_failing.clone.return_value.apply_async.side_effect = (
- RuntimeError("broker outage")
- )
-
- with pytest.raises(RuntimeError, match=r"broker outage"):
- barrier.enqueue(
- [h1, h2, h3_failing],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
-
- # Initial SET + EXPIRE happened, then DEL on cleanup.
- # The DEL call should reference BOTH barrier keys.
- delete_calls = redis_mock.delete.call_args_list
- # At least one of the DEL calls must include both barrier keys
- # (the cleanup call), distinguishable from the initial
- # ``delete(_results_key(...))`` stale-list clear.
- cleanup_args = [
- set(call.args) for call in delete_calls
- if len(call.args) == 2
- ]
- assert any(
- {_remaining_key("exec-1"), _results_key("exec-1")} == args
- for args in cleanup_args
- ), (
- f"Expected DEL(remaining, results) in cleanup; saw "
- f"delete calls: {[call.args for call in delete_calls]}"
- )
-
-
-# --- link_error: barrier_abort ---
-
-
-class TestBarrierAbortLinkError:
- """``link_error`` propagates header-task failures.
-
- Without this, a failed header task would leave the counter stuck
- above 0 and the execution would hang until TTL. ``barrier_abort``
- DELs the barrier keys + logs the error; the outer task's error
- handler is responsible for the workflow status update.
- """
-
- def test_abort_registered_under_canonical_name(self):
- """Mixed-version rolling deploys require the link_error task
- to be importable by name on every worker that processes it."""
- from queue_backend import barrier_abort
-
- assert barrier_abort.name == "barrier_abort"
-
- def test_link_error_attached_to_each_header_task(self, barrier, redis_mock):
- """Every header task must get ``barrier_abort.s(...)`` as a
- ``.link_error``. Without this, header failures don't
- propagate and the execution hangs."""
- h1 = MagicMock(name="h1")
- h2 = MagicMock(name="h2")
- barrier.enqueue(
- [h1, h2],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-1"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
- h1.clone.return_value.link_error.assert_called_once()
- h2.clone.return_value.link_error.assert_called_once()
-
- @patch("queue_backend.redis_barrier._get_redis_client")
- def test_abort_deletes_both_barrier_keys(self, get_redis):
- """``barrier_abort`` cleans up the barrier state on header
- failure — DELs both keys. The first call to ``barrier_abort``
- for a given execution_id wins the SET NX race and proceeds
- with cleanup + log."""
- from queue_backend import barrier_abort
-
- redis_client = MagicMock(name="redis")
- # ``set(...nx=True)`` returns True on first SET — this abort wins.
- redis_client.set.return_value = True
- get_redis.return_value = redis_client
-
- barrier_abort.run(
- MagicMock(name="request"),
- RuntimeError("header task crashed"),
- "traceback string",
- execution_id="exec-42",
- )
-
- redis_client.delete.assert_called_once_with(
- _remaining_key("exec-42"),
- _results_key("exec-42"),
- )
-
- @patch("queue_backend.redis_barrier._get_redis_client")
- def test_concurrent_aborts_deduplicate(self, get_redis):
- """N concurrent header-task failures attach the same
- ``link_error`` → N ``barrier_abort`` executions for one
- logical execution failure. Without dedup that would produce
- N ``logger.error`` calls and N Sentry events. The SET NX
- lock makes the first abort win; subsequent aborts early-exit
- silently."""
- from queue_backend import barrier_abort
-
- redis_client = MagicMock(name="redis")
- # First call: SET NX returns True (first abort wins).
- # Second call: SET NX returns False (lock already exists).
- redis_client.set.side_effect = [True, False]
- get_redis.return_value = redis_client
-
- result_1 = barrier_abort.run(
- MagicMock(name="request"),
- RuntimeError("header task 1 crashed"),
- "traceback string",
- execution_id="exec-42",
- )
- result_2 = barrier_abort.run(
- MagicMock(name="request"),
- RuntimeError("header task 2 crashed"),
- "traceback string",
- execution_id="exec-42",
- )
-
- # First abort completed cleanup; second deduplicated.
- assert result_1["status"] == "aborted"
- assert result_2["status"] == "deduplicated"
-
- # DELETE called exactly ONCE — only the winning abort cleans
- # up. The deduplicated path early-exits before DELETE.
- redis_client.delete.assert_called_once_with(
- _remaining_key("exec-42"),
- _results_key("exec-42"),
- )
-
- @patch("queue_backend.redis_barrier._get_redis_client")
- def test_abort_uses_setnx_with_ttl(self, get_redis):
- """The SET NX lock has an EX (TTL) — the lock key auto-
- cleans alongside the barrier keys instead of leaking
- forever on a failed execution."""
- from queue_backend import barrier_abort
- from queue_backend.redis_barrier import _KEY_TTL_DEFAULT_SECONDS
-
- redis_client = MagicMock(name="redis")
- redis_client.set.return_value = True
- get_redis.return_value = redis_client
-
- barrier_abort.run(
- MagicMock(name="request"),
- RuntimeError("crash"),
- "traceback",
- execution_id="exec-1",
- )
-
- # SET called with nx=True + ex=<ttl>.
- redis_client.set.assert_called_once()
- call = redis_client.set.call_args
- assert call.args[0] == "barrier:abort_lock:exec-1"
- assert call.kwargs.get("nx") is True
- assert call.kwargs.get("ex") == _KEY_TTL_DEFAULT_SECONDS
-
- @patch("queue_backend.redis_barrier._get_redis_client")
- def test_abort_delete_failure_releases_lock(self, get_redis):
- """If SET NX succeeds (this abort wins) but the subsequent
- DELETE then raises (mid-abort Redis blip), the lock MUST be
- released — otherwise sibling aborts hit the dedup early-exit
- while the barrier keys are still alive, in-flight successful
- tasks DECR to 0, and the callback fires with partial results
- (silent-success-on-mid-abort-failure).
-
- Same correctness class as the stale-lock-on-retry bug,
- reached via a different path: mid-abort failure instead of
- cross-execution stale state.
- """
- from queue_backend import barrier_abort
- from queue_backend.redis_barrier import _abort_lock_key
-
- redis_client = MagicMock(name="redis")
- # SET NX returns True (this abort wins), DELETE then raises.
- redis_client.set.return_value = True
- # Two delete calls expected: the barrier-keys DEL that raises,
- # then the lock-release DEL.
- redis_client.delete.side_effect = [
- ConnectionError("redis blip mid-abort"), # first DELETE raises
- 1, # lock-release DEL succeeds (returns count of deleted keys)
- ]
- get_redis.return_value = redis_client
-
- with pytest.raises(ConnectionError, match=r"redis blip mid-abort"):
- barrier_abort.run(
- MagicMock(name="request"),
- RuntimeError("header task crashed"),
- "traceback",
- execution_id="exec-1",
- )
-
- # First DELETE was the barrier-keys cleanup; SECOND DELETE was
- # the lock release (so siblings can re-acquire).
- delete_calls = redis_client.delete.call_args_list
- assert len(delete_calls) == 2, (
- f"Expected 2 delete calls (barrier keys + lock release); "
- f"got {len(delete_calls)}: {delete_calls}"
- )
- # The second call must target the abort_lock specifically.
- assert delete_calls[1].args == (_abort_lock_key("exec-1"),), (
- f"Expected lock-release DELETE; got args={delete_calls[1].args!r}"
- )
-
-
-# --- Retry with reused execution_id: stale abort_lock cleanup ---
-
-
-class TestAbortLockRetryCleanup:
- """A retry that reuses ``execution_id`` must not be poisoned by a
- stale ``abort_lock`` left behind by the previous failed run.
-
- Failure mode without the fix: execution A fails → ``barrier_abort``
- writes ``abort_lock:foo`` (TTL 6h) → within 6h, execution B retries
- with ``exec_id=foo`` → ``enqueue`` resets remaining/results but
- abort_lock survives → if any header in B fails, its ``barrier_abort``
- hits the stale lock and early-exits without DELing B's keys →
- in-flight successful tasks DECR ``remaining`` to 0 → callback fires
- with partial results → failed retry silently masked as successful.
-
- Fix: ``enqueue`` DELs the abort_lock alongside the results list at
- setup time, so a retry starts with a clean slate.
- """
-
- def test_enqueue_clears_stale_abort_lock(self, barrier, redis_mock):
- """``enqueue`` must DEL the abort_lock key at setup so a
- stale lock from a previous (failed) execution with the same
- execution_id doesn't poison the retry's abort path."""
- from queue_backend.redis_barrier import _abort_lock_key
-
- barrier.enqueue(
- [MagicMock(name="h1")],
- callback_task_name="cb",
- callback_kwargs={"execution_id": "exec-foo"},
- callback_queue="q",
- app_instance=MagicMock(),
- )
- # The setup DEL must include the abort_lock key (alongside
- # results) — otherwise a stale lock from a prior run with the
- # same execution_id survives into this enqueue.
- redis_mock.delete.assert_called_once_with(
- _results_key("exec-foo"),
- _abort_lock_key("exec-foo"),
- )
-
-
-# --- Process-local Redis client caching ---
-
-
-class TestRedisClientSingleton:
- """``_get_redis_client()`` returns a process-cached client so the
- high-frequency ``barrier_decr_and_check`` / ``barrier_abort``
- invocations reuse the same ``ConnectionPool`` rather than spinning
- up a fresh pool (and TCP handshakes) per task. At 1000-task
- fan-out the difference is 1000 vs 1 pool initialisation.
- """
-
- def test_get_redis_client_returns_same_instance(self):
- """Repeated calls within one process must return the same
- ``Redis`` instance — guarantees connection-pool reuse."""
- from queue_backend import redis_barrier
-
- # Reset the cache so this test exercises the lazy-init path.
- redis_barrier._redis_client_singleton = None
- try:
- with patch(
- "queue_backend.redis_barrier.create_redis_client"
- ) as factory:
- factory.return_value = MagicMock(name="redis_client")
- client_a = redis_barrier._get_redis_client()
- client_b = redis_barrier._get_redis_client()
- client_c = redis_barrier._get_redis_client()
-
- # All three calls return the same object.
- assert client_a is client_b is client_c
- # ``create_redis_client`` factory called exactly ONCE —
- # subsequent calls hit the cache.
- assert factory.call_count == 1
- finally:
- # Reset for downstream tests.
- redis_barrier._redis_client_singleton = None
-
- def test_falls_back_to_canonical_redis_prefix_for_sentinel_ssl_inheritance(
- self, monkeypatch
- ):
- """When no ``WORKER_BARRIER_REDIS_HOST`` is set, the barrier
- MUST use the canonical ``REDIS_`` prefix directly so it
- inherits the FULL config including ``REDIS_SENTINEL_MODE`` and
- ``REDIS_SSL`` — not just HOST/PORT/PASSWORD/USER/DB.
-
- Without this fix, a Sentinel-backed deployment that relies on
- the documented fallback would connect standalone (Sentinel
- mode is per-prefix in ``create_redis_client`` and doesn't
- cross-fall-back), failing every execution at flag-flip time."""
- from queue_backend import redis_barrier
-
- # Ensure no barrier-specific HOST is set.
- monkeypatch.delenv("WORKER_BARRIER_REDIS_HOST", raising=False)
- redis_barrier._redis_client_singleton = None
- try:
- with patch(
- "queue_backend.redis_barrier.create_redis_client"
- ) as factory:
- factory.return_value = MagicMock(name="redis_client")
- redis_barrier._get_redis_client()
-
- # The factory MUST be called with env_prefix="REDIS_" so
- # SENTINEL_MODE/SSL inherit from the canonical instance.
- factory.assert_called_once()
- assert factory.call_args.kwargs.get("env_prefix") == "REDIS_", (
- f"Expected env_prefix='REDIS_' when no barrier-specific "
- f"HOST is set (so Sentinel/SSL inherit); got "
- f"env_prefix={factory.call_args.kwargs.get('env_prefix')!r}"
- )
- finally:
- redis_barrier._redis_client_singleton = None
-
- def test_uses_barrier_prefix_when_dedicated_host_set(self, monkeypatch):
- """When ``WORKER_BARRIER_REDIS_HOST`` IS set, the operator has
- opted into a dedicated barrier Redis and is responsible for
- the full barrier-prefix config (HOST + PORT + ... +
- SENTINEL_MODE + SSL)."""
- from queue_backend import redis_barrier
-
- monkeypatch.setenv("WORKER_BARRIER_REDIS_HOST", "dedicated-redis")
- redis_barrier._redis_client_singleton = None
- try:
- with patch(
- "queue_backend.redis_barrier.create_redis_client"
- ) as factory:
- factory.return_value = MagicMock(name="redis_client")
- redis_barrier._get_redis_client()
-
- assert (
- factory.call_args.kwargs.get("env_prefix")
- == "WORKER_BARRIER_REDIS_"
- ), (
- f"Expected env_prefix='WORKER_BARRIER_REDIS_' when "
- f"dedicated HOST is set; got "
- f"env_prefix={factory.call_args.kwargs.get('env_prefix')!r}"
- )
- finally:
- redis_barrier._redis_client_singleton = None
-
-
-if __name__ == "__main__":
- pytest.main([__file__, "-v"])
diff --git a/workers/tests/test_simple_prompt_studio_operation.py b/workers/tests/test_simple_prompt_studio_operation.py
index 926430e23..75dfef21e 100644
--- a/workers/tests/test_simple_prompt_studio_operation.py
+++ b/workers/tests/test_simple_prompt_studio_operation.py
@@ -2,13 +2,14 @@
executor registers, routes to its own queue, and LegacyExecutor rejects them.
"""
-from unittest.mock import MagicMock
from unstract.sdk1.execution.context import ExecutionContext
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.sdk1.execution.executor import BaseExecutor
from unstract.sdk1.execution.registry import ExecutorRegistry
from unstract.sdk1.execution.result import ExecutionResult
+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher
+
+from .executor_dispatch_fakes import FakeExecutorTransport, queue_for
# ---------------------------------------------------------------------------
# Mock SimplePromptStudioExecutor — registration and execution
@@ -103,18 +104,16 @@ class TestSimplePromptStudioRegistration:
class TestSPSQueueRouting:
def test_sps_routes_to_correct_queue(self):
- queue = ExecutionDispatcher._get_queue("simple_prompt_studio")
+ queue = queue_for("simple_prompt_studio")
assert queue == "celery_executor_simple_prompt_studio"
def test_dispatch_sends_to_sps_queue(self):
- mock_app = MagicMock()
- mock_result = MagicMock()
- mock_result.get.return_value = ExecutionResult(
- success=True, data={"output": {"field": "value"}}
- ).to_dict()
- mock_app.send_task.return_value = mock_result
-
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
+ transport = FakeExecutorTransport(
+ result=ExecutionResult(
+ success=True, data={"output": {"field": "value"}}
+ ).to_dict()
+ )
+ dispatcher = PgExecutionDispatcher(transport)
ctx = ExecutionContext(
executor_name="simple_prompt_studio",
operation="sps_answer_prompt",
@@ -124,19 +123,15 @@ class TestSPSQueueRouting:
)
dispatcher.dispatch(ctx)
- mock_app.send_task.assert_called_once()
- call_kwargs = mock_app.send_task.call_args
- assert call_kwargs.kwargs.get("queue") == "celery_executor_simple_prompt_studio"
+ assert transport.queue == "celery_executor_simple_prompt_studio"
def test_dispatch_sps_index_to_correct_queue(self, tmp_path):
- mock_app = MagicMock()
- mock_result = MagicMock()
- mock_result.get.return_value = ExecutionResult(
- success=True, data={"output": "indexed"}
- ).to_dict()
- mock_app.send_task.return_value = mock_result
-
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
+ transport = FakeExecutorTransport(
+ result=ExecutionResult(
+ success=True, data={"output": "indexed"}
+ ).to_dict()
+ )
+ dispatcher = PgExecutionDispatcher(transport)
ctx = ExecutionContext(
executor_name="simple_prompt_studio",
operation="sps_index",
@@ -146,9 +141,7 @@ class TestSPSQueueRouting:
)
dispatcher.dispatch(ctx)
- mock_app.send_task.assert_called_once()
- call_kwargs = mock_app.send_task.call_args
- assert call_kwargs.kwargs.get("queue") == "celery_executor_simple_prompt_studio"
+ assert transport.queue == "celery_executor_simple_prompt_studio"
# ---------------------------------------------------------------------------
diff --git a/workers/tests/test_smart_table_extract_operation.py b/workers/tests/test_smart_table_extract_operation.py
index 608e5c8b7..971d2dbe6 100644
--- a/workers/tests/test_smart_table_extract_operation.py
+++ b/workers/tests/test_smart_table_extract_operation.py
@@ -2,13 +2,14 @@
its own queue, and LegacyExecutor rejects the operation.
"""
-from unittest.mock import MagicMock
from unstract.sdk1.execution.context import ExecutionContext
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.sdk1.execution.executor import BaseExecutor
from unstract.sdk1.execution.registry import ExecutorRegistry
from unstract.sdk1.execution.result import ExecutionResult
+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher
+
+from .executor_dispatch_fakes import FakeExecutorTransport, queue_for
# ---------------------------------------------------------------------------
# tasks.py log_component for smart_table_extract
@@ -117,18 +118,16 @@ class TestSmartTableExtractorRegistration:
class TestSmartTableQueueRouting:
def test_smart_table_routes_to_correct_queue(self):
- queue = ExecutionDispatcher._get_queue("smart_table")
+ queue = queue_for("smart_table")
assert queue == "celery_executor_smart_table"
def test_dispatch_sends_to_smart_table_queue(self):
- mock_app = MagicMock()
- mock_result = MagicMock()
- mock_result.get.return_value = ExecutionResult(
- success=True, data={"output": "ok"}
- ).to_dict()
- mock_app.send_task.return_value = mock_result
-
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
+ transport = FakeExecutorTransport(
+ result=ExecutionResult(
+ success=True, data={"output": "ok"}
+ ).to_dict()
+ )
+ dispatcher = PgExecutionDispatcher(transport)
ctx = ExecutionContext(
executor_name="smart_table",
operation="smart_table_extract",
@@ -138,9 +137,7 @@ class TestSmartTableQueueRouting:
)
dispatcher.dispatch(ctx)
- mock_app.send_task.assert_called_once()
- call_kwargs = mock_app.send_task.call_args
- assert call_kwargs.kwargs.get("queue") == "celery_executor_smart_table"
+ assert transport.queue == "celery_executor_smart_table"
# ---------------------------------------------------------------------------
diff --git a/workers/tests/test_summarize_operation.py b/workers/tests/test_summarize_operation.py
index 4308387cc..9e8828e26 100644
--- a/workers/tests/test_summarize_operation.py
+++ b/workers/tests/test_summarize_operation.py
@@ -3,18 +3,20 @@
Verifies:
1. Summarize operation exists and routes through LegacyExecutor
2. Summarize executor_params contract matches _handle_summarize expectations
-3. Dispatch routes summarize to celery_executor_legacy queue
+3. Dispatch routes summarize to celery_executor_legacy queue (PG executor RPC)
4. Summarize result has expected shape (data.data = summary text)
-5. Full Celery chain for summarize operation
+5. Full dispatch chain for summarize operation
"""
from unittest.mock import MagicMock, patch
import pytest
from unstract.sdk1.execution.context import ExecutionContext, Operation
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.sdk1.execution.registry import ExecutorRegistry
from unstract.sdk1.execution.result import ExecutionResult
+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher
+
+from .executor_dispatch_fakes import FakeExecutorTransport, queue_for
# Patches
_PATCH_GET_PROMPT_DEPS = (
@@ -80,18 +82,16 @@ class TestSummarizeParamsContract:
class TestSummarizeQueueRouting:
def test_summarize_routes_to_legacy_queue(self):
"""Summarize dispatches to celery_executor_legacy (LegacyExecutor)."""
- queue = ExecutionDispatcher._get_queue("legacy")
+ queue = queue_for("legacy")
assert queue == "celery_executor_legacy"
def test_dispatch_sends_summarize_to_legacy_queue(self):
- mock_app = MagicMock()
- mock_result = MagicMock()
- mock_result.get.return_value = ExecutionResult(
- success=True, data={"data": "Summary text here"}
- ).to_dict()
- mock_app.send_task.return_value = mock_result
-
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
+ transport = FakeExecutorTransport(
+ result=ExecutionResult(
+ success=True, data={"data": "Summary text here"}
+ ).to_dict()
+ )
+ dispatcher = PgExecutionDispatcher(transport)
ctx = ExecutionContext(
executor_name="legacy",
operation="summarize",
@@ -108,9 +108,7 @@ class TestSummarizeQueueRouting:
)
result = dispatcher.dispatch(ctx)
- mock_app.send_task.assert_called_once()
- call_kwargs = mock_app.send_task.call_args
- assert call_kwargs.kwargs.get("queue") == "celery_executor_legacy"
+ assert transport.queue == "celery_executor_legacy"
assert result.success
assert result.data["data"] == "Summary text here"
diff --git a/workers/tests/test_table_extract_operation.py b/workers/tests/test_table_extract_operation.py
index 916921298..01d659f0e 100644
--- a/workers/tests/test_table_extract_operation.py
+++ b/workers/tests/test_table_extract_operation.py
@@ -2,12 +2,13 @@
routes to its own queue, and LegacyExecutor rejects the operation.
"""
-from unittest.mock import MagicMock
from unstract.sdk1.execution.context import ExecutionContext
-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher
from unstract.sdk1.execution.registry import ExecutorRegistry
from unstract.sdk1.execution.result import ExecutionResult
+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher
+
+from .executor_dispatch_fakes import FakeExecutorTransport, queue_for
# ---------------------------------------------------------------------------
# tasks.py log_component for table_extract
@@ -122,19 +123,17 @@ class TestTableExtractorExecutorRegistration:
class TestTableQueueRouting:
def test_table_executor_routes_to_correct_queue(self):
"""executor_name='table' routes to celery_executor_table queue."""
- queue = ExecutionDispatcher._get_queue("table")
+ queue = queue_for("table")
assert queue == "celery_executor_table"
def test_dispatch_sends_to_table_queue(self):
- """ExecutionDispatcher sends table_extract to correct queue."""
- mock_app = MagicMock()
- mock_result = MagicMock()
- mock_result.get.return_value = ExecutionResult(
- success=True, data={"output": "ok"}
- ).to_dict()
- mock_app.send_task.return_value = mock_result
-
- dispatcher = ExecutionDispatcher(celery_app=mock_app)
+ """The dispatcher sends table_extract to the correct queue."""
+ transport = FakeExecutorTransport(
+ result=ExecutionResult(
+ success=True, data={"output": "ok"}
+ ).to_dict()
+ )
+ dispatcher = PgExecutionDispatcher(transport)
ctx = ExecutionContext(
executor_name="table",
operation="table_extract",
@@ -144,9 +143,7 @@ class TestTableQueueRouting:
)
dispatcher.dispatch(ctx)
- mock_app.send_task.assert_called_once()
- call_kwargs = mock_app.send_task.call_args
- assert call_kwargs.kwargs.get("queue") == "celery_executor_table"
+ assert transport.queue == "celery_executor_table"
# ---------------------------------------------------------------------------
@@ -193,4 +190,4 @@ class TestEntryPointConfig:
# This is a documentation/verification test — the entry point
# in pyproject.toml maps 'table' to TableExtractorExecutor.
# Verify the queue name matches.
- assert ExecutionDispatcher._get_queue("table") == "celery_executor_table"
+ assert queue_for("table") == "celery_executor_table"
diff --git a/workers/tests/test_terminal_execution_guard.py b/workers/tests/test_terminal_execution_guard.py
index bc301979b..dc334bbbe 100644
--- a/workers/tests/test_terminal_execution_guard.py
+++ b/workers/tests/test_terminal_execution_guard.py
@@ -38,38 +38,36 @@ ACTIVE = [ExecutionStatus.PENDING.value, ExecutionStatus.EXECUTING.value]
@pytest.mark.parametrize("status", TERMINAL)
-def test_guard_raises_for_terminal_on_pg(status):
+def test_guard_raises_for_terminal(status):
with pytest.raises(_TerminalExecutionSkip) as exc:
- _raise_if_execution_terminal({"status": status}, "exec-1", is_pg=True)
+ _raise_if_execution_terminal({"status": status}, "exec-1")
assert exc.value.execution_id == "exec-1"
assert exc.value.status == status
@pytest.mark.parametrize("status", ACTIVE)
-def test_guard_passes_for_active_on_pg(status):
+def test_guard_passes_for_active(status):
# Must NOT raise for PENDING/EXECUTING.
- _raise_if_execution_terminal({"status": status}, "exec-1", is_pg=True)
+ _raise_if_execution_terminal({"status": status}, "exec-1")
[email protected]("status", TERMINAL)
-def test_guard_is_noop_on_celery_path(status):
- # is_pg=False (Celery) → NEVER raises, even for terminal statuses. This is
- # what keeps the Celery flow behaviorally unchanged.
- _raise_if_execution_terminal({"status": status}, "exec-1", is_pg=False)
+# ``test_guard_is_noop_on_celery_path`` is gone with the transport: the guard
+# used to be is_pg-gated so the Celery flow never raised. It is unconditional
+# now — the parametrised raise/pass tests above are the whole behaviour.
-def test_missing_status_on_pg_warns_and_proceeds():
+def test_missing_status_warns_and_proceeds():
# A missing status is fail-open (no raise) but surfaced as a warning, since
# it signals a degraded execution-fetch response rather than an active run.
with mock.patch("file_processing.tasks.logger") as log:
- _raise_if_execution_terminal({}, "exec-1", is_pg=True)
+ _raise_if_execution_terminal({}, "exec-1")
log.warning.assert_called_once()
def test_guard_fires_at_real_call_site_before_status_write():
- """Exercises the real wiring the mocked-setup tests skip: the is_pg forward
- chain and the guard's placement in _setup_execution_context — after the
- execution fetch, before the EXECUTING status write."""
+ """Exercises the real wiring the mocked-setup tests skip: the guard's
+ placement in _setup_execution_context — after the execution fetch, before
+ the EXECUTING status write."""
api_client = mock.Mock()
api_client.get_workflow_execution.return_value = types.SimpleNamespace(
success=True, data={"execution": {"status": ExecutionStatus.ERROR.value}}
@@ -85,7 +83,7 @@ def test_guard_fires_at_real_call_site_before_status_write():
mock.patch("file_processing.tasks.create_organization_context"),
pytest.raises(_TerminalExecutionSkip),
):
- _setup_execution_context(batch_data, "task-1", is_pg=True)
+ _setup_execution_context(batch_data, "task-1")
# The guard fired BEFORE the EXECUTING status write.
api_client.update_workflow_execution_status.assert_not_called()
@@ -127,7 +125,7 @@ def test_run_batch_stages_skips_terminal_without_processing():
"file_processing.tasks._process_individual_files"
) as process_files,
):
- result = _run_batch_stages({"any": "payload"}, "task-1", is_pg=True)
+ result = _run_batch_stages({"any": "payload"}, "task-1")
pre_create.assert_not_called()
process_files.assert_not_called()
@@ -197,7 +195,7 @@ def test_run_batch_stages_proceeds_when_not_terminal():
return_value={"total_files": 1, "successful_files": 1, "failed_files": 0},
),
):
- result = _run_batch_stages({"any": "payload"}, "task-1", is_pg=False)
+ result = _run_batch_stages({"any": "payload"}, "task-1")
pre_create.assert_called_once()
process_files.assert_called_once()
diff --git a/workers/tests/test_workflow_context_transport.py b/workers/tests/test_workflow_context_transport.py
deleted file mode 100644
index b8db8d643..000000000
--- a/workers/tests/test_workflow_context_transport.py
+++ /dev/null
@@ -1,51 +0,0 @@
-"""Tests for the 9e transport field carried on ``WorkflowContextData``.
-
-The transport a workflow execution rides is decided once at creation and
-carried in the task payload; on the worker side it lands on the live
-``WorkflowContextData`` so the fan-out (PR 2) can read it. PR 1 only adds the
-field with a Celery default — these tests pin that it defaults and round-trips.
-"""
-
-from unittest.mock import MagicMock
-
-from shared.models.execution_models import (
- WorkerOrganizationContext,
- WorkflowContextData,
-)
-
-from unstract.core.data_models import DEFAULT_WORKFLOW_TRANSPORT, WorkflowTransport
-
-
-def _make_context(**overrides):
- org_context = WorkerOrganizationContext(
- organization_id="org-1",
- api_client=MagicMock(),
- )
- kwargs = {
- "workflow_id": "wf-1",
- "workflow_name": "wf-name",
- "workflow_type": "TASK",
- "execution_id": "exec-1",
- "organization_context": org_context,
- "files": {},
- }
- kwargs.update(overrides)
- return WorkflowContextData(**kwargs)
-
-
-class TestWorkflowContextTransport:
- def test_defaults_to_celery(self):
- """A context built without a transport rides legacy Celery."""
- assert _make_context().transport == DEFAULT_WORKFLOW_TRANSPORT
- assert _make_context().transport == WorkflowTransport.CELERY.value
-
- def test_carries_pg_queue_when_set(self):
- ctx = _make_context(transport=WorkflowTransport.PG_QUEUE.value)
- assert ctx.transport == "pg_queue"
-
- def test_invalid_transport_coerced_to_celery(self):
- """A garbage payload value (version skew / bad PG JSONB) must fail closed
- at construction, not reach the PR 2 fan-out read."""
- assert _make_context(transport="pg-queue").transport == "celery"
- assert _make_context(transport="bogus").transport == "celery"
- assert _make_context(transport=None).transport == "celery"