LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3522 [FEAT] PG Queue Phase 6a — Barrier interface + CeleryChordBarrier wrapper + lift chord call sites + plumb fairness (#2024)
ali8592fe6427b9
diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.pyindex 4a0e2683d..9f87c2c6a 100644--- a/workers/api-deployment/tasks.py+++ b/workers/api-deployment/tasks.py@@ -7,7 +7,8 @@ Uses the same patterns as workflow_helper.py and file_execution_tasks.pyimport timefrom typing import Any-from queue_backend import worker_task+from queue_backend import FairnessKey, dispatch, worker_task+from queue_backend.fairness import WorkloadTypefrom shared.api import InternalAPIClientfrom shared.enums.status_enums import PipelineStatusfrom shared.enums.task_enums import TaskName@@ -668,22 +669,89 @@ def _run_workflow_api(# Create callback queue using same logic as Django backendfile_processing_callback_queue = _get_callback_queue_name_api()- # Execute chord exactly matching Django pattern- from celery import chord-- result = chord(batch_tasks)(- app.signature(- "process_batch_callback_api", # Use API-specific callback- kwargs={- "execution_id": str(execution_id),- "pipeline_id": str(pipeline_id) if pipeline_id else None,- "organization_id": str(schema_name),- }, # Pass required parameters for API callback- queue=file_processing_callback_queue,- )+ # Route through ``WorkflowOrchestrationUtils.create_chord_execution``+ # (which delegates to ``CeleryChordBarrier`` under the hood) instead of+ # calling ``chord(...)`` directly. Same Celery semantics, but the+ # ``Barrier`` abstraction lets a future Phase 6b swap to+ # ``RedisDecrBarrier`` behind a flag without touching this call site.+ # The header tasks + callback now carry the fairness slot for+ # downstream PG Queue routing — closes the chord-fairness gap noted+ # in Phase 5.1.+ result = WorkflowOrchestrationUtils.create_chord_execution(+ batch_tasks=batch_tasks,+ callback_task_name="process_batch_callback_api",+ callback_kwargs={+ "execution_id": str(execution_id),+ "pipeline_id": str(pipeline_id) if pipeline_id else None,+ "organization_id": str(schema_name),+ },+ callback_queue=file_processing_callback_queue,+ app_instance=app,+ fairness=FairnessKey(+ org_id=str(schema_name),+ workload_type=WorkloadType.API,+ ),)if not result:+ # Two reasons ``result`` can be falsy:+ # 1. Zero ``batch_tasks`` — the barrier short-circuits and+ # returns ``None``. Pre-Barrier this would have called+ # ``chord(empty)(callback)`` which Celery handles by+ # firing the callback immediately with ``[]``. We+ # preserve that contract byte-for-byte by dispatching+ # the callback explicitly with an empty result list, so+ # workflow_execution_status, API result cache, and+ # pipeline notifications all run exactly as they would+ # have pre-Barrier. (Unreachable in practice — upstream+ # requires non-empty ``hash_values_of_files`` — but+ # the defence+ # closes the theoretical gap.)+ # 2. ``batch_tasks`` is non-empty but the barrier returned+ # falsy for some other reason — a genuine queue failure.+ if not batch_tasks:+ logger.info(+ f"Execution {execution_id} orchestrated with zero "+ "batch tasks — firing callback directly with empty "+ "result list (preserves pre-Barrier zero-files contract)"+ )+ # Route through ``queue_backend.dispatch`` (not raw+ # ``app.send_task``) so the canary in+ # ``test_dispatch_sites_characterisation.py`` stays+ # happy and the fairness slot rides on the wire same+ # as the chord-path callback does. body=[] matches+ # Celery's chord-with-empty-header semantic exactly.+ callback_result = dispatch(+ "process_batch_callback_api",+ args=[[]],+ kwargs={+ "execution_id": str(execution_id),+ "pipeline_id": str(pipeline_id) if pipeline_id else None,+ "organization_id": str(schema_name),+ },+ queue=file_processing_callback_queue,+ fairness=FairnessKey(+ org_id=str(schema_name),+ workload_type=WorkloadType.API,+ ),+ )+ return {+ "status": "orchestrated",+ "execution_id": execution_id,+ "workflow_id": workflow_id,+ "task_id": task_id,+ "files_processed": total_files,+ "files_from_cache": len(cached_results),+ "batches_created": 0,+ "chord_id": callback_result.id,+ "cached_results": list(cached_results.keys())+ if cached_results+ else [],+ "message": (+ f"Zero batch tasks for execution {execution_id} — "+ "callback fired directly with empty result list"+ ),+ }exception = f"Failed to queue execution task {execution_id}"logger.error(exception)raise Exception(exception)diff --git a/workers/general/tasks.py b/workers/general/tasks.pyindex aa784c81f..edceacdb7 100644--- a/workers/general/tasks.py+++ b/workers/general/tasks.py@@ -7,7 +7,8 @@ and general workflow executions using internal APIs.import timefrom typing import Any-from queue_backend import worker_task+from queue_backend import FairnessKey, worker_task+from queue_backend.fairness import WorkloadTypefrom scheduler.tasks import execute_pipeline_task_v2# Import shared worker infrastructure using new structure@@ -937,13 +938,20 @@ def _orchestrate_file_processing_general(# Import to ensure we have the right app contextfrom worker import app as celery_app- # Use shared orchestration utility for chord execution+ # Route through the ``Barrier``-backed helper. The general path+ # is ETL / non-API workflow execution — ``NON_API`` workload type+ # so any future PG Queue fairness scheduler can split API traffic+ # from background workflow processing.result = WorkflowOrchestrationUtils.create_chord_execution(batch_tasks=batch_tasks,callback_task_name=TaskName.PROCESS_BATCH_CALLBACK.value,callback_kwargs=callback_kwargs,callback_queue=file_processing_callback_queue,app_instance=celery_app,+ fairness=FairnessKey(+ org_id=organization_id,+ workload_type=WorkloadType.NON_API,+ ),)if not result:diff --git a/workers/queue_backend/__init__.py b/workers/queue_backend/__init__.pyindex a8a0d681b..2e7395135 100644--- a/workers/queue_backend/__init__.py+++ b/workers/queue_backend/__init__.py@@ -4,8 +4,16 @@ Single place where the substrate choice (Celery today; PG Queue later)lives. Both entry points are transparent passthroughs to Celery today."""+from .barrier import Barrier, BarrierHandle, CeleryChordBarrierfrom .decorator import worker_taskfrom .dispatch import dispatchfrom .fairness import FairnessKey-__all__ = ["FairnessKey", "dispatch", "worker_task"]+__all__ = [+ "Barrier",+ "BarrierHandle",+ "CeleryChordBarrier",+ "FairnessKey",+ "dispatch",+ "worker_task",+]diff --git a/workers/queue_backend/barrier.py b/workers/queue_backend/barrier.pynew file mode 100644index 000000000..13c37e520--- /dev/null+++ b/workers/queue_backend/barrier.py@@ -0,0 +1,196 @@+"""Distributed barrier — abstraction over the chord-completion pattern.++The two production chord call sites (``shared/workflow/execution/+orchestration_utils.py`` and ``api-deployment/tasks.py``) need to wait+for a fan-out of header tasks to all complete, then fire a callback+with the aggregated results. Today that's implemented by Celery's+``chord(header)(body)`` primitive, which is the highest-risk Celery+construct at our scale — silent task drops at ~130K-task scale are+documented in the PG Queue decision journey.++This module provides the ``Barrier`` protocol and a ``CeleryChordBarrier``+implementation that wraps the existing ``chord(...)`` call 1:1. Lifting+both call sites to ``Barrier`` gives Phase 6b a place to land an+alternative implementation (e.g. ``RedisDecrBarrier`` using the labs+``DECR remaining:{exec_id}`` pattern, or — later in Phase 8 — a+``PgBarrier`` using SKIP LOCKED) without touching the call sites a+second time.++**Phase 6a (this PR): wrapper only — behaviour-preserving uplift.**+``CeleryChordBarrier.enqueue(...)`` produces byte-identical wire output+to the direct ``chord(...)`` calls it replaces *when ``fairness=None``*.+When a ``FairnessKey`` is passed (which both production call sites+now do), each header task and the callback additionally carry an+``x-fairness-key`` AMQP header. That's an additive wire-level change —+consumers ignore unknown headers — same posture as Phase 5.1's+``dispatch()`` plumbing. Mixed-version rolling deploys are safe in+both directions: old workers ignore the new header; new workers+handle messages without the header identically to today.++Phase 6b (separate PR) will add ``RedisDecrBarrier`` as a second+implementation, gated by ``WORKER_BARRIER_BACKEND`` env flag (default+``chord``). Phase 6c is the rollout.+"""++from __future__ import annotations++import logging+from typing import TYPE_CHECKING, Any, Protocol++from celery import chord++from .fairness import FairnessKey+from .handle import BarrierHandle++if TYPE_CHECKING:+ from celery.canvas import Signature++logger = logging.getLogger(__name__)+++class Barrier(Protocol):+ """Fan-out-then-callback primitive.++ Semantically: "enqueue these header tasks in parallel; when all+ complete, fire this callback with the list of header results."+ The concrete substrate (Celery chord today, Redis DECR or PG+ SKIP LOCKED later) is hidden behind this protocol.+ """++ def enqueue(+ self,+ header_tasks: list[Signature],+ *,+ callback_task_name: str,+ callback_kwargs: dict[str, Any],+ callback_queue: str,+ app_instance: Any,+ fairness: FairnessKey | None = None,+ ) -> BarrierHandle | None:+ """Enqueue ``header_tasks`` and a single callback to fire on completion.++ ``None`` is the **sole signal** that no work was enqueued — it+ is returned exclusively when ``header_tasks`` is empty. Any+ substrate-level failure (broker outage, serialisation error,+ etc.) raises rather than returning ``None``, so callers may+ treat ``None`` as "no-op, nothing wrong" and a raised+ exception as a genuine failure. The caller is responsible for+ guarding ``header_tasks`` (e.g. an early-return when files==0)+ if a returned ``None`` is semantically distinct from a normal+ completion in their domain.++ ``fairness``, when provided, is attached as the+ ``x-fairness-key`` header on every enqueued task and the+ callback — same wire shape as ``dispatch()`` uses for bare+ sites. Pass ``None`` for non-workflow-execution callers.++ ``app_instance`` is a Celery app today (the Protocol leaks+ this concrete dependency for the same reason the+ ``CeleryChordBarrier`` does — Phase 6b's RedisDecrBarrier will+ narrow this to a smaller Protocol). The argument is the+ ``app`` from the producer's task context and must expose+ ``.signature(name, kwargs=..., queue=..., headers=...)``.+ """+ ...+++class CeleryChordBarrier:+ """Thin wrapper around ``celery.chord``.++ Behaviour-preserving: the ``chord(header_tasks)(callback_signature)``+ call is byte-identical to the inline ``chord(...)`` it replaces.+ The only additional behaviour is the optional ``fairness`` header+ attachment, which is an additive wire-level change (consumers that+ don't read the header are unaffected — same backward-compat+ posture as Phase 5.1's ``dispatch()`` fairness plumbing).++ Future ``RedisDecrBarrier`` and ``PgBarrier`` implementations will+ live alongside this class and be selected by the+ ``WORKER_BARRIER_BACKEND`` env flag (Phase 6b).+ """++ def enqueue(+ self,+ header_tasks: list[Signature],+ *,+ callback_task_name: str,+ callback_kwargs: dict[str, Any],+ callback_queue: str,+ app_instance: Any,+ fairness: FairnessKey | None = None,+ ) -> BarrierHandle | None:+ """See :class:`Barrier.enqueue`."""+ # Empty-header guard goes FIRST so a zero-task run skips+ # signature construction / fairness serialisation entirely.+ # Any failure in those paths is now constrained to the+ # non-empty branch where the work is actually needed.+ if not header_tasks:+ execution_id = callback_kwargs.get("execution_id")+ pipeline_id = callback_kwargs.get("pipeline_id")+ logger.info(+ f"[exec:{execution_id}] [pipeline:{pipeline_id}] "+ "Zero header tasks detected — skipping barrier enqueue "+ "(parent should handle pipeline status updates directly)"+ )+ return None++ try:+ # Use ``fairness.as_header()`` — the single wire-shape+ # encoder, also used by ``dispatch.py``. Constructing the+ # header dict inline here would risk silent divergence+ # from the bare-dispatch path.+ fairness_headers = fairness.as_header() if fairness else None++ # Build the callback-signature kwargs explicitly. When+ # ``fairness=None`` the resulting call shape is+ # byte-identical to the pre-Barrier ``app.signature(name,+ # kwargs=..., queue=...)`` (no spurious ``headers=None``).+ signature_kwargs: dict[str, Any] = {+ "kwargs": callback_kwargs,+ "queue": callback_queue,+ }+ if fairness_headers:+ signature_kwargs["headers"] = fairness_headers+ callback_signature = app_instance.signature(+ callback_task_name, **signature_kwargs+ )++ # Stamp fairness onto each header signature via+ # ``Signature.clone().set(headers=...)`` rather than+ # in-place ``.set(...)`` on the caller's list. Cloning+ # avoids cross-tenant header leakage if a future retry+ # path or signature cache ever re-uses the original+ # ``header_tasks`` list with a different ``FairnessKey``.+ if fairness_headers:+ signed_header_tasks = [+ task.clone().set(headers=fairness_headers) for task in header_tasks+ ]+ else:+ signed_header_tasks = header_tasks++ result = chord(signed_header_tasks)(callback_signature)++ logger.info(+ f"Barrier enqueued via CeleryChordBarrier — "+ f"header_tasks={len(signed_header_tasks)}, "+ f"callback={callback_task_name}, "+ f"queue={callback_queue}"+ )++ return result++ except Exception:+ # ``logger.exception`` auto-attaches the traceback — needed+ # for debugging broker outages / serialisation failures+ # at the chord entry point. Add the readily-available+ # context so a Sentry/log triage gets execution/pipeline/+ # callback/queue identifiers alongside the stack.+ execution_id = callback_kwargs.get("execution_id")+ pipeline_id = callback_kwargs.get("pipeline_id")+ logger.exception(+ f"[exec:{execution_id}] [pipeline:{pipeline_id}] "+ f"Failed to enqueue barrier "+ f"(callback={callback_task_name}, queue={callback_queue}, "+ f"header_tasks={len(header_tasks)})"+ )+ raisediff --git a/workers/queue_backend/dispatch.py b/workers/queue_backend/dispatch.pyindex 21f0ea68c..cf8d2c274 100644--- a/workers/queue_backend/dispatch.py+++ b/workers/queue_backend/dispatch.py@@ -8,21 +8,12 @@ call sites.from __future__ import annotationsfrom collections.abc import Mapping, Sequence-from typing import Any, Protocol+from typing import Anyfrom celery import current_appfrom .fairness import FairnessKey---class DispatchHandle(Protocol):- """Minimum contract every dispatch substrate must satisfy.-- Celery's ``AsyncResult`` satisfies this via ``.id``; any future- substrate handle must expose the same attribute.- """-- id: str+from .handle import DispatchHandledef dispatch(diff --git a/workers/queue_backend/handle.py b/workers/queue_backend/handle.pynew file mode 100644index 000000000..d5b20f5bd--- /dev/null+++ b/workers/queue_backend/handle.py@@ -0,0 +1,37 @@+"""Shared task-handle protocol.++Single ``TaskHandle`` contract every ``queue_backend`` entry point+returns: ``dispatch()`` (bare task) and ``Barrier.enqueue()`` (chord /+future DECR / future PG-SKIP-LOCKED). Celery's ``AsyncResult``+satisfies this via ``.id``; future non-Celery substrate handles must+expose the same attribute so call sites that log ``chord_id`` /+``task_id`` keep working.++Originally split as separate ``DispatchHandle`` and ``BarrierHandle``+Protocols — they were byte-for-byte identical, which Vishnu's review+on PR #2024 correctly flagged as a drift risk. Consolidated into+``TaskHandle`` here; both old names are kept as documentation aliases+for now.+"""++from __future__ import annotations++from typing import Protocol+++class TaskHandle(Protocol):+ """Return-value contract for any ``queue_backend`` enqueue path.++ Celery's ``AsyncResult`` satisfies this via ``.id``. A future+ non-Celery substrate handle must expose the same attribute so+ call sites that log ``chord_id`` / ``task_id`` keep working.+ """++ id: str+++# Documentation aliases. Kept so existing imports and docstring+# references don't break, but they all refer to the same single+# Protocol — no more drift risk.+DispatchHandle = TaskHandle+BarrierHandle = TaskHandlediff --git a/workers/shared/workflow/execution/orchestration_utils.py b/workers/shared/workflow/execution/orchestration_utils.pyindex e70930a1d..6d4cf7ff4 100644--- a/workers/shared/workflow/execution/orchestration_utils.py+++ b/workers/shared/workflow/execution/orchestration_utils.py@@ -7,7 +7,7 @@ chord execution, batch processing, and task coordination utilities.import osfrom typing import Any-from celery import chord+from queue_backend import CeleryChordBarrier, FairnessKeyfrom ...enums import FileDestinationType, PipelineTypefrom ...enums.worker_enums import QueueName@@ -15,6 +15,12 @@ from ...infrastructure.logging import WorkerLoggerlogger = WorkerLogger.get_logger(__name__)+# Single ``Barrier`` instance reused across all+# ``WorkflowOrchestrationUtils.create_chord_execution`` calls. A future+# Phase 6b will replace this with a factory call (``get_barrier()``)+# that picks the impl from ``WORKER_BARRIER_BACKEND``.+_BARRIER = CeleryChordBarrier()+class WorkflowOrchestrationUtils:"""Centralized workflow orchestration patterns and utilities."""@@ -26,8 +32,16 @@ class WorkflowOrchestrationUtils:callback_kwargs: dict[str, Any],callback_queue: str,app_instance: Any,+ *,+ fairness: FairnessKey | None = None,) -> Any:- """Standardized chord creation and execution pattern.+ """Standardized fan-out + callback pattern (Phase 6 ``Barrier``).++ Routes through ``CeleryChordBarrier`` — a thin wrapper around+ ``celery.chord(header)(body)`` that lets Phase 6b swap the+ substrate (e.g. to ``RedisDecrBarrier``) without touching this+ call site a second time. Behaviour is identical to the+ previous direct ``chord(...)`` call.Args:batch_tasks: List of batch task signatures@@ -35,49 +49,29 @@ class WorkflowOrchestrationUtils:callback_kwargs: Keyword arguments for callbackcallback_queue: Queue name for callback taskapp_instance: Celery app instance+ fairness: Optional ``FairnessKey`` attached as a message+ header on every header task and the callback — closes+ the chord-fairness gap that Phase 5.1 (``dispatch()``)+ deliberately scoped out. See ``queue_backend.fairness``+ for the slot name constant.Returns:- Chord result object or None if no batch tasks+ Barrier handle (Celery ``AsyncResult``-shaped — ``.id``+ exposed for chord-id logging) or None if no batch tasks.Note:- This consolidates the identical chord creation pattern found in- api-deployment and general workers.-- CRITICAL: Returns None for zero batch tasks, signaling to parent- that direct pipeline status updates should be handled instead.+ CRITICAL: Returns None for zero batch tasks, signaling to+ parent that direct pipeline status updates should be+ handled instead."""- try:- callback_signature = app_instance.signature(- callback_task_name,- kwargs=callback_kwargs,- queue=callback_queue,- )- # For zero files, skip chord entirely - parent should handle status updates directly- if not batch_tasks:- # Extract execution_id from callback kwargs for logging- execution_id = callback_kwargs.get("execution_id")- pipeline_id = callback_kwargs.get("pipeline_id")- logger.info(- f"[exec:{execution_id}] [pipeline:{pipeline_id}] Zero batch tasks detected - skipping chord execution "- f"(parent should handle pipeline status updates directly)"- )- return None # Signal to parent that no chord was created-- # Normal chord execution for non-empty batch tasks- result = chord(batch_tasks)(callback_signature)-- logger.info(- f"Chord execution started - "- f"batch_tasks={len(batch_tasks)}, "- f"callback={callback_task_name}, "- f"queue={callback_queue}"- )-- return result-- except Exception as e:- logger.error(f"Failed to create chord execution: {e}")- raise+ return _BARRIER.enqueue(+ batch_tasks,+ callback_task_name=callback_task_name,+ callback_kwargs=callback_kwargs,+ callback_queue=callback_queue,+ app_instance=app_instance,+ fairness=fairness,+ )@staticmethoddef determine_manual_review_routing(@@ -325,16 +319,33 @@ class WorkflowOrchestrationMixin:"""Mixin class to add orchestration utilities to worker tasks."""def create_chord(- self, batch_tasks, callback_task_name, callback_kwargs, callback_queue+ self,+ batch_tasks,+ callback_task_name,+ callback_kwargs,+ callback_queue,+ *,+ fairness: FairnessKey | None = None,):- """Create chord using standardized pattern."""+ """Create chord using standardized pattern.++ Forwards ``fairness`` to ``create_chord_execution`` so mixin+ callers can opt into the chord-fairness plumbing that bare+ ``dispatch()`` sites get via Phase 5.1. ``None`` keeps the+ pre-Barrier wire shape exactly.+ """# Get app instance from task contextapp_instance = getattr(self, "app", None)if not app_instance:raise RuntimeError("Celery app instance not available in task context")return WorkflowOrchestrationUtils.create_chord_execution(- batch_tasks, callback_task_name, callback_kwargs, callback_queue, app_instance+ batch_tasks,+ callback_task_name,+ callback_kwargs,+ callback_queue,+ app_instance,+ fairness=fairness,)def determine_manual_review_routing(self, files, manual_review_config=None):diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.pynew file mode 100644index 000000000..a9ae7fa36--- /dev/null+++ b/workers/tests/test_barrier.py@@ -0,0 +1,461 @@+"""Characterisation tests for the ``Barrier`` abstraction (PG Queue Phase 6a).++The two production chord call sites+(``WorkflowOrchestrationUtils.create_chord_execution`` and+``api-deployment/tasks.py``'s inline chord) were lifted behind+``CeleryChordBarrier`` so a future Phase 6b can swap in+``RedisDecrBarrier`` (or PG-based) implementations without touching+call sites.++Three layers of coverage:++1. **Barrier protocol shape** — ``CeleryChordBarrier`` satisfies the+ ``Barrier`` Protocol and ``AsyncResult`` satisfies ``BarrierHandle``.+2. **Wire equivalence with the pre-Barrier chord call** — given the+ same args, ``CeleryChordBarrier.enqueue(...)`` produces the same+ ``chord(header)(body)`` invocation, the same ``app.signature(...)``+ args, and the same return value. A regression that bypasses the+ abstraction or alters the wire surface fails one of these tests.+3. **Fairness plumbing** — when ``fairness=`` is passed, the fairness+ header is attached to every enqueued task and to the callback+ (closes Phase 5.1's chord-fairness gap that was deliberately+ scoped out).++The single ``chord(...)`` call still lives in+``queue_backend/barrier.py`` (asserted by the inventory test in+``test_chord_sites_characterisation.py``).+"""++from __future__ import annotations++from unittest.mock import MagicMock, patch++import pytest++from queue_backend import (+ Barrier,+ BarrierHandle,+ CeleryChordBarrier,+ FairnessKey,+)+from queue_backend.fairness import FAIRNESS_HEADER_NAME, WorkloadType+++def app() -> MagicMock:+ """Celery-app-shaped mock with a working ``.signature(...)``.++ Used by every test that drives ``CeleryChordBarrier.enqueue(...)``.+ Shared via fixture rather than a per-class ``_make_app`` helper to+ keep test classes free of boilerplate (SonarCloud S4144+ duplication).+ """+ mock = MagicMock(name="celery_app")+ mock.signature.return_value = MagicMock(name="callback_signature")+ return mock+++def mock_chord():+ """Patch ``queue_backend.barrier.chord`` and yield the mock.++ Every wire-equivalence / fairness test patches the same import+ target — extracting to a fixture removes ~8 identical+ ``with patch(...) as mock_chord:`` lines (SonarCloud S4144+ duplication).+ """+ with patch("queue_backend.barrier.chord") as m:+ m.return_value = MagicMock(name="chord_object")+ yield m+++# --- Protocol shape ---+++class TestBarrierProtocolShape:+ def test_celery_chord_barrier_satisfies_protocol(self):+ """``CeleryChordBarrier`` is structurally a ``Barrier``.++ Pinned so Phase 6b's ``RedisDecrBarrier`` can be substituted at+ any call site without runtime ``AttributeError``.+ """+ barrier: Barrier = CeleryChordBarrier()+ assert callable(getattr(barrier, "enqueue", None))++ def test_barrier_handle_protocol_is_minimal(self):+ """``BarrierHandle`` only requires ``id: str`` — celery's+ ``AsyncResult`` satisfies this. Future substrate handles must+ too so existing chord-id logging keeps working."""+ # MagicMock with .id attribute is structurally a BarrierHandle.+ handle = MagicMock()+ handle.id = "task-1"+ # Treating it as a BarrierHandle at runtime works (Python's+ # Protocols are duck-typed; this test pins the contract).+ h: BarrierHandle = handle+ assert h.id == "task-1"+++# --- Wire equivalence with the pre-Barrier chord call ---+++class TestCeleryChordBarrierWireEquivalence:+ """``CeleryChordBarrier.enqueue(...)`` must produce the same+ ``chord(batch_tasks)(callback_signature)`` invocation that the+ original inline ``chord(...)`` calls produced — modulo the+ additive ``x-fairness-key`` header that the barrier optionally+ stamps onto signatures.+ """++ def test_empty_header_returns_none_and_skips_chord(self, app, mock_chord):+ """Zero-batch contract preserved: barrier returns None,+ ``chord(...)`` is never called.++ Parent callers (``general/tasks.py``, ``api-deployment/tasks.py``)+ rely on this signal to handle pipeline status updates directly.+ """+ result = CeleryChordBarrier().enqueue(+ [],+ callback_task_name="process_batch_callback",+ callback_kwargs={"execution_id": "exec-1", "pipeline_id": "pipe-1"},+ callback_queue="file_processing_callback",+ app_instance=app,+ )++ assert result is None+ mock_chord.assert_not_called()++ def test_non_empty_header_invokes_chord_header_then_body(self, app, mock_chord):+ """The original ``chord(header)(body)`` two-step call must+ be preserved exactly — a refactor that flattens or reorders+ these calls would change Celery's chord semantics."""+ header = [MagicMock(name="h1"), MagicMock(name="h2")]++ CeleryChordBarrier().enqueue(+ header,+ callback_task_name="process_batch_callback",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="file_processing_callback",+ app_instance=app,+ )++ # Step 1: chord(header)+ mock_chord.assert_called_once_with(header)+ # Step 2: chord_obj(callback_signature) — applies the chord+ mock_chord.return_value.assert_called_once_with(app.signature.return_value)++ def test_callback_signature_args_match_pre_uplift_contract(self, app, mock_chord):+ """``app.signature(task_name, kwargs=..., queue=...)`` is+ called with exactly the args the pre-Barrier helper used."""+ callback_kwargs = {+ "execution_id": "exec-42",+ "pipeline_id": "pipe-7",+ "organization_id": "org-x",+ }++ CeleryChordBarrier().enqueue(+ [MagicMock(name="h")],+ callback_task_name="process_batch_callback_api",+ callback_kwargs=callback_kwargs,+ callback_queue="api_file_processing_callback",+ app_instance=app,+ )++ # Without ``fairness=``, no ``headers=`` kwarg on the signature+ # call — matches the pre-Barrier wire byte-for-byte.+ app.signature.assert_called_once_with(+ "process_batch_callback_api",+ kwargs=callback_kwargs,+ queue="api_file_processing_callback",+ )++ def test_returns_chord_result_object(self, app, mock_chord):+ """The barrier handle is whatever ``chord(header)(body)``+ returns — Celery's ``AsyncResult`` in production. Callers+ log ``.id`` for chord-id tracing."""+ chord_result = MagicMock(name="chord_result")+ mock_chord.return_value.return_value = chord_result++ result = CeleryChordBarrier().enqueue(+ [MagicMock()],+ callback_task_name="process_batch_callback",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="file_processing_callback",+ app_instance=app,+ )++ assert result is chord_result++ def test_chord_failure_is_re_raised_after_logging(self, app, mock_chord):+ """If ``chord(...)`` raises (broker outage, serialisation+ error, etc.), the barrier logs and re-raises — never swallows.++ Without this, callers would treat broker failures as silent+ zero-task chords and skip pipeline status updates entirely.+ """+ mock_chord.side_effect = RuntimeError("broker exploded")++ with pytest.raises(RuntimeError, match="broker exploded"):+ CeleryChordBarrier().enqueue(+ [MagicMock()],+ callback_task_name="process_batch_callback",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="file_processing_callback",+ app_instance=app,+ )+++# --- Fairness plumbing ---+++class TestCeleryChordBarrierFairnessHeader:+ """When ``fairness=FairnessKey(...)`` is passed, the barrier+ stamps ``x-fairness-key`` on every header task and the callback.++ Closes the chord-fairness gap from Phase 5.1: bare ``dispatch()``+ calls carried fairness; the two chord call sites didn't (they+ bypassed ``dispatch()`` entirely). After this Phase 6a uplift,+ every queue-crossing payload on the workflow-execution path now+ carries the fairness slot.+ """++ def test_fairness_header_stamped_on_callback_signature(self, app, mock_chord):+ CeleryChordBarrier().enqueue(+ [MagicMock(name="h")],+ callback_task_name="process_batch_callback_api",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="api_file_processing_callback",+ app_instance=app,+ fairness=FairnessKey(org_id="org-1", workload_type=WorkloadType.API),+ )++ headers = app.signature.call_args.kwargs.get("headers")+ assert headers is not None, (+ "expected ``headers=`` kwarg on app.signature when fairness "+ "is passed"+ )+ assert headers[FAIRNESS_HEADER_NAME] == {+ "org_id": "org-1",+ "workload_type": "api",+ "pipeline_priority": 5,+ }++ def test_fairness_header_stamped_on_every_header_task(self, app, mock_chord):+ """Every batch task in the header carries the fairness header+ so PG Queue's per-task fairness scheduler can route each+ independently.++ Header signatures are stamped via ``Signature.clone().set(...)``+ (not in-place ``.set(...)``) — clone avoids cross-tenant+ header leakage if a future retry path or signature cache ever+ re-uses the original ``header_tasks`` list with a different+ ``FairnessKey``. The test asserts ``.clone().set(headers=...)``+ is called on each original task.+ """+ h1, h2, h3 = (MagicMock(name=f"header_task_{i}") for i in range(3))++ CeleryChordBarrier().enqueue(+ [h1, h2, h3],+ callback_task_name="process_batch_callback",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="file_processing_callback",+ app_instance=app,+ fairness=FairnessKey(org_id="org-1", workload_type=WorkloadType.NON_API),+ )++ expected = {+ FAIRNESS_HEADER_NAME: {+ "org_id": "org-1",+ "workload_type": "non_api",+ "pipeline_priority": 5,+ }+ }+ for task in (h1, h2, h3):+ # ``.clone()`` produces a fresh signature; the ``.set(...)``+ # then attaches the fairness header to the clone, leaving+ # the original ``task`` unchanged.+ task.clone.assert_called_once_with()+ task.clone.return_value.set.assert_called_once_with(headers=expected)+ # Direct ``.set(...)`` on the original is NEVER called —+ # the whole point of the clone-and-set pattern.+ task.set.assert_not_called()++ def test_no_fairness_no_header_added(self, app, mock_chord):+ """When ``fairness=None``, the barrier behaves byte-for-byte+ like the pre-Barrier chord call — no ``headers=`` kwarg on+ the signature, no ``.set(headers=...)`` on header tasks.++ Mixed-version rolling deploys are safe because this branch+ produces the same wire as the old direct chord call.+ """+ header = [MagicMock(name="h1"), MagicMock(name="h2")]++ CeleryChordBarrier().enqueue(+ header,+ callback_task_name="process_batch_callback",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="file_processing_callback",+ app_instance=app,+ )++ # Callback signature: no headers kwarg.+ assert "headers" not in app.signature.call_args.kwargs++ # Header tasks: ``.set(...)`` not called for headers stamping.+ for task in header:+ # ``set`` might be called for other reasons in production,+ # but never with ``headers=`` when fairness is None.+ for call in task.set.call_args_list:+ assert "headers" not in call.kwargs, (+ "unexpected fairness header stamped without fairness="+ )+++# --- Singleton routing (pins the _BARRIER dispatch point) ---+++class TestOrchestrationUtilsRoutesThroughSingleton:+ """Pin the ``WorkflowOrchestrationUtils.create_chord_execution``+ routing to the module-level ``_BARRIER`` singleton.++ A refactor that bypasses the singleton (e.g. inlining+ ``CeleryChordBarrier().enqueue(...)`` inside the static method,+ or going back to a direct ``chord(...)`` call) would still pass+ the inventory canary in ``test_chord_sites_characterisation.py``+ as long as ``chord(...)`` lives somewhere inside ``barrier.py``.+ This test closes that gap so the singleton is the unambiguous+ single dispatch point — ready for Phase 6b's factory swap.+ """++ def test_create_chord_execution_delegates_to_module_barrier(self):+ from shared.workflow.execution import orchestration_utils+ from shared.workflow.execution.orchestration_utils import (+ WorkflowOrchestrationUtils,+ )++ app_mock = MagicMock(name="celery_app")+ app_mock.signature.return_value = MagicMock(name="callback_signature")+ batch = [MagicMock(name="h1")]+ fairness = FairnessKey(org_id="org-1", workload_type=WorkloadType.API)++ with patch.object(orchestration_utils, "_BARRIER") as mock_barrier:+ mock_barrier.enqueue.return_value = MagicMock(name="result")+ WorkflowOrchestrationUtils.create_chord_execution(+ batch_tasks=batch,+ callback_task_name="process_batch_callback_api",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="api_file_processing_callback",+ app_instance=app_mock,+ fairness=fairness,+ )++ mock_barrier.enqueue.assert_called_once_with(+ batch,+ callback_task_name="process_batch_callback_api",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="api_file_processing_callback",+ app_instance=app_mock,+ fairness=fairness,+ )+++# --- Call-site fairness contracts ---+++class TestCallSiteFairnessContracts:+ """Pin the ``FairnessKey`` shape each production call site declares.++ A refactor that swaps the workload types, drops the ``fairness=``+ kwarg, or transposes the org-id source goes undetected by the+ isolated barrier tests above. These tests assert the contract at+ the call site boundary.+ """++ def test_api_deployment_declares_api_workload_with_schema_name(self):+ """``api-deployment/tasks.py`` must pass ``WorkloadType.API``+ and ``org_id=str(schema_name)``.++ ``api-deployment`` is not a valid Python identifier (the dash+ prevents ``import api-deployment.tasks``), so the test reads+ the source file by path instead of importing the module.+ """+ import inspect+ import pathlib++ # Derive workers root from a sibling importable module+ # (``queue_backend.barrier``) so the path stays correct under+ # any test runner working directory.+ src = inspect.getfile(__import__("queue_backend.barrier", fromlist=["a"]))+ api_tasks_path = (+ pathlib.Path(src).parent.parent / "api-deployment" / "tasks.py"+ )+ text = api_tasks_path.read_text()++ # Assert the FairnessKey block at the chord call site declares+ # the contract this PR's commit message claims it does.+ assert "fairness=FairnessKey(" in text+ assert "workload_type=WorkloadType.API" in text+ assert "org_id=str(schema_name)" in text++ def test_general_declares_non_api_workload_with_organization_id(self):+ """``general/tasks.py`` must pass ``WorkloadType.NON_API`` and+ ``org_id=organization_id``."""+ import inspect+ import pathlib++ src = inspect.getfile(__import__("queue_backend.barrier", fromlist=["a"]))+ general_tasks_path = (+ pathlib.Path(src).parent.parent / "general" / "tasks.py"+ )+ text = general_tasks_path.read_text()++ assert "fairness=FairnessKey(" in text+ assert "workload_type=WorkloadType.NON_API" in text+ assert "org_id=organization_id" in text+++# --- Zero-files contract (regression pin for T1) ---+++class TestApiDeploymentZeroFilesContract:+ """Pin the api-deployment zero-files handler.++ Before the Barrier uplift, ``chord(empty)(callback)`` returned a+ truthy ``AsyncResult`` (Celery fires the body immediately with+ ``[]``). The Barrier returns ``None`` for empty headers, so the+ caller now has to handle the ``None`` case explicitly — otherwise+ the existing ``if not result: raise`` would map zero-files runs+ to ``ExecutionStatus.ERROR``.++ The post-Barrier handler explicitly dispatches the callback with+ an empty result list to preserve pre-Barrier behaviour+ byte-for-byte: ``workflow_execution_status`` updates, API result+ caching, and pipeline notifications all run exactly as they would+ have pre-Barrier. Unreachable in practice (upstream guarantees+ non-empty ``hash_values_of_files``) but this test pins the defensive+ contract so a future refactor doesn't silently regress.+ """++ def test_api_deployment_zero_files_dispatches_callback_with_empty_list(self):+ import inspect+ import pathlib++ src = inspect.getfile(__import__("queue_backend.barrier", fromlist=["a"]))+ api_tasks_path = (+ pathlib.Path(src).parent.parent / "api-deployment" / "tasks.py"+ )+ text = api_tasks_path.read_text()++ # Defensive branch present and dispatches the callback via+ # the seam (queue_backend.dispatch), not raw send_task — the+ # dispatch-sites canary in test_dispatch_sites_characterisation.py+ # enforces that. The literal strings are part of the+ # externally-observable response + behaviour contract.+ assert "if not batch_tasks:" in text+ assert "dispatch(" in text # routed through queue_backend seam+ assert '"process_batch_callback_api"' in text # callback fired+ assert "args=[[]]" in text # body=[] matches chord-empty semantic+ assert '"batches_created": 0' in text+++if __name__ == "__main__":+ pytest.main([__file__, "-v"])diff --git a/workers/tests/test_chord_sites_characterisation.py b/workers/tests/test_chord_sites_characterisation.pyindex 7c9d653c2..79a8202e6 100644--- a/workers/tests/test_chord_sites_characterisation.py+++ b/workers/tests/test_chord_sites_characterisation.py@@ -1,21 +1,27 @@-"""Characterisation tests for the two ``celery.chord`` call sites.--These are the two places in ``workers/`` source that invoke ``chord(...)``-directly. PR #13 will replace both with a transport-agnostic ``Barrier``-abstraction matching the labs target architecture's ``DECR remaining:{exec_id}``-pattern.--This test suite locks down the **current** chord invocation contract so the-migration can be proved equivalent. Chord is the highest-risk Celery construct-called out in the PG Queue decision doc (silent task drops at ~130K-task scale)-— characterising it before refactor is critical.--Sites:-1. ``shared/workflow/execution/orchestration_utils.py`` — ``WorkflowOrchestrationUtils.create_chord_execution`` (the centralised helper).-2. ``api-deployment/tasks.py`` — inline chord inside ``_run_workflow_api``.- Site 2 is exercised only via the inventory test (full characterisation- requires heavy mocking of the enclosing 273-line function — out of scope- for a smoke/characterisation pass).+"""Characterisation tests for the chord-callback boundary, now lifted+behind the ``Barrier`` abstraction (PG Queue Phase 6a).++Originally written to lock down the inline ``celery.chord(...)`` calls+at two call sites before they were lifted. Post-uplift the inline+calls are gone — both call sites delegate to+``WorkflowOrchestrationUtils.create_chord_execution`` which in turn+delegates to ``CeleryChordBarrier.enqueue(...)``. The single remaining+``chord(...)`` call lives inside ``workers/queue_backend/barrier.py``.++Sites covered (assertions unchanged from the original pre-uplift+suite — they characterise the **same** ``chord(header)(body)``+contract, now reached one indirection later):+1. ``shared/workflow/execution/orchestration_utils.py`` —+ ``WorkflowOrchestrationUtils.create_chord_execution`` (centralised helper).+2. ``api-deployment/tasks.py`` — same helper, called inline inside+ ``_run_workflow_api`` (post-uplift it no longer calls ``chord(...)``+ directly; covered by the inventory test below).++Chord is the highest-risk Celery construct called out in the PG Queue+decision doc (silent task drops at ~130K-task scale). A future+Phase 6b will introduce ``RedisDecrBarrier`` as a second implementation+behind a flag; these tests stay green through that transition because+they characterise the **Celery chord** code path specifically."""from unittest.mock import MagicMock, patch@@ -49,9 +55,7 @@ class TestCreateChordExecution:app = self._make_app_instance()- with patch(- "shared.workflow.execution.orchestration_utils.chord"- ) as mock_chord:+ with patch("queue_backend.barrier.chord") as mock_chord:result = WorkflowOrchestrationUtils.create_chord_execution(batch_tasks=[],callback_task_name="process_batch_callback",@@ -72,9 +76,7 @@ class TestCreateChordExecution:app = self._make_app_instance()batch_tasks = [MagicMock(name="batch_task_1"), MagicMock(name="batch_task_2")]- with patch(- "shared.workflow.execution.orchestration_utils.chord"- ) as mock_chord:+ with patch("queue_backend.barrier.chord") as mock_chord:# chord(batch_tasks) returns a chord object; calling it with the# callback signature returns the chord result. Both calls must# happen for this characterisation.@@ -109,9 +111,7 @@ class TestCreateChordExecution:"organization_id": "org-x",}- with patch(- "shared.workflow.execution.orchestration_utils.chord"- ) as mock_chord:+ with patch("queue_backend.barrier.chord") as mock_chord:mock_chord.return_value = MagicMock()WorkflowOrchestrationUtils.create_chord_execution(batch_tasks=batch_tasks,@@ -136,9 +136,7 @@ class TestCreateChordExecution:app = self._make_app_instance()- with patch(- "shared.workflow.execution.orchestration_utils.chord"- ) as mock_chord:+ with patch("queue_backend.barrier.chord") as mock_chord:chord_obj = MagicMock()chord_result = MagicMock(name="chord_result_object")chord_obj.return_value = chord_result@@ -162,9 +160,7 @@ class TestCreateChordExecution:app = self._make_app_instance()- with patch(- "shared.workflow.execution.orchestration_utils.chord"- ) as mock_chord:+ with patch("queue_backend.barrier.chord") as mock_chord:mock_chord.side_effect = RuntimeError("broker exploded")with pytest.raises(RuntimeError, match="broker exploded"):@@ -215,12 +211,16 @@ class TestWorkflowOrchestrationMixinCreateChord:callback_queue="file_processing_callback",)+ # Mixin now forwards ``fairness=`` (None when caller omits it)+ # to ``create_chord_execution``. The positional args stay the+ # same.mock_static.assert_called_once_with(batch,"process_batch_callback",kwargs,"file_processing_callback",task.app,+ fairness=None,)def test_create_chord_raises_when_no_app_bound(self):@@ -247,14 +247,37 @@ class TestWorkflowOrchestrationMixinCreateChord:class TestChordSiteInventory:- """Exactly two chord call sites must exist in workers/ source.-- If a third appears, this test fails so PR #13's migration can't silently- miss it.+ """Post-Barrier-uplift invariant: exactly ONE ``chord(...)`` call+ site, inside ``queue_backend/barrier.py``.++ Before the uplift there were two inline ``chord(...)`` calls (in+ ``orchestration_utils.py`` and ``api-deployment/tasks.py``). The+ refactor lifted both behind ``CeleryChordBarrier`` so the chord+ primitive has a single home — a future ``RedisDecrBarrier`` swap+ can target one location instead of chasing call sites.++ If a third ``chord(...)`` call appears anywhere outside+ ``barrier.py``, this test fails — preventing a silent regression+ where a future PR bypasses the abstraction."""- def test_only_two_known_chord_call_sites_in_workers(self):- """Count chord(...) invocations (not imports) in workers/ source."""+ _CHORD_HOST = "queue_backend/barrier.py"++ def test_chord_invocation_lives_only_in_barrier(self):+ """Count chord(...) invocations (not imports) in workers/ source.++ Tightened regex (after the original ``(?:^|[\\s=(])chord\\(``+ could match docstring prose mentioning ``chord(...)`` after+ whitespace): now requires assignment form ``= chord(`` (with+ optional whitespace), which matches the production call at+ ``barrier.py``'s ``result = chord(header_tasks)(callback_signature)``+ but rejects docstring / comment mentions like ``the chord(...)+ primitive``.++ Comment / docstring lines are also explicitly skipped as+ belt-and-suspenders against a future call site that lacks the+ ``=`` prefix (e.g. a return-value-discarded ``chord(...)``).+ """import pathlibimport re@@ -264,12 +287,10 @@ class TestChordSiteInventory:# `workers/shared/tests_helpers/`.skip_top_dirs = {"tests", "__pycache__", "htmlcov", ".venv"}- # Match `chord(` as a function call — excludes the bare import line- # `from celery import chord` and helper method names like `create_chord`.- # The regex requires `chord` to be preceded by start-of-line / whitespace- # / `=` / `(` (i.e., a true call expression), not as part of a longer- # identifier such as `create_chord`.- pattern = re.compile(r"(?:^|[\s=(])chord\(")+ # Assignment-form ``= chord(`` — production call sites all+ # capture the AsyncResult into a variable. Docstring prose+ # never uses this form.+ pattern = re.compile(r"=\s*chord\(")hits = []for py in workers_root.rglob("*.py"):@@ -278,21 +299,31 @@ class TestChordSiteInventory:continuetext = py.read_text()for line_no, line in enumerate(text.splitlines(), start=1):+ stripped = line.lstrip()+ # Skip pure comment lines (belt-and-suspenders — the+ # regex already rejects most of these via the ``=``+ # prefix requirement, but a comment like ``# x = chord(...)``+ # would otherwise sneak through).+ if stripped.startswith("#"):+ continueif pattern.search(line):hits.append(f"{py.relative_to(workers_root)}:{line_no}")- # Expected exactly two — in orchestration_utils.py and api-deployment/tasks.py.- assert len(hits) == 2, (- f"Expected exactly 2 chord(...) call sites in workers/, found "- f"{len(hits)}:\n " + "\n ".join(hits)+ assert len(hits) == 1, (+ "Expected exactly 1 chord(...) call site in workers/ (inside "+ f"{self._CHORD_HOST}), found {len(hits)}:\n "+ + "\n ".join(hits)+ + "\nPhase 6a lifted both call sites behind CeleryChordBarrier — "+ "a new direct chord(...) call bypasses the Barrier abstraction "+ "and breaks the future Phase 6b RedisDecrBarrier swap point."+ )+ assert self._CHORD_HOST in hits[0], (+ f"chord(...) call must live in {self._CHORD_HOST}, found at {hits[0]}")- joined = " ".join(hits)- assert "shared/workflow/execution/orchestration_utils.py" in joined- assert "api-deployment/tasks.py" in joined- def test_chord_import_only_in_two_files(self):- """`from celery import chord` should appear in exactly the two files- that actually invoke chord — no other imports lurking."""+ def test_chord_import_only_in_barrier(self):+ """`from celery import chord` should appear in exactly one file —+ ``queue_backend/barrier.py`` — post-Barrier uplift."""import pathlibimport re@@ -311,16 +342,13 @@ class TestChordSiteInventory:if pattern.search(line):hits.append(f"{py.relative_to(workers_root)}:{line_no}")- assert len(hits) == 2, (- f"Expected `from celery import chord` in exactly 2 files, found "+ assert len(hits) == 1, (+ f"Expected `from celery import chord` in exactly 1 file, found "f"{len(hits)}:\n " + "\n ".join(hits))- # Sanity: same files as the call-site canary above. If the imports- # ever migrate to different files while count stays at 2, this catches- # it — preventing a silent miss during the Barrier migration.- joined = " ".join(hits)- assert "shared/workflow/execution/orchestration_utils.py" in joined- assert "api-deployment/tasks.py" in joined+ assert self._CHORD_HOST in hits[0], (+ f"chord import must live in {self._CHORD_HOST}, found at {hits[0]}"+ )if __name__ == "__main__":diff --git a/workers/tests/test_queue_backend_seam.py b/workers/tests/test_queue_backend_seam.pyindex 5a6ac88b5..7075dbaf2 100644--- a/workers/tests/test_queue_backend_seam.py+++ b/workers/tests/test_queue_backend_seam.py@@ -275,7 +275,17 @@ class TestPublicSurface:def test_all_exports(self):import queue_backend- assert set(queue_backend.__all__) == {"FairnessKey", "dispatch", "worker_task"}+ # Phase 6a added Barrier / BarrierHandle / CeleryChordBarrier.+ # Phase 6b will add a ``get_barrier`` factory when the+ # ``WORKER_BARRIER_BACKEND`` flag wiring lands.+ assert set(queue_backend.__all__) == {+ "Barrier",+ "BarrierHandle",+ "CeleryChordBarrier",+ "FairnessKey",+ "dispatch",+ "worker_task",+ }if __name__ == "__main__":