Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3522 [FEAT] PG Queue Phase 6a — Barrier interface + CeleryChordBarrier wrapper + lift chord call sites + plumb fairness (#2024)

ali8592fe6427b9
diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py
index 4a0e2683d..9f87c2c6a 100644
--- a/workers/api-deployment/tasks.py
+++ b/workers/api-deployment/tasks.py
@@ -7,7 +7,8 @@ Uses the same patterns as workflow_helper.py and file_execution_tasks.py
import time
from typing import Any
-from queue_backend import worker_task
+from queue_backend import FairnessKey, dispatch, worker_task
+from queue_backend.fairness import WorkloadType
from shared.api import InternalAPIClient
from shared.enums.status_enums import PipelineStatus
from shared.enums.task_enums import TaskName
@@ -668,22 +669,89 @@ def _run_workflow_api(
# Create callback queue using same logic as Django backend
file_processing_callback_queue = _get_callback_queue_name_api()
- # Execute chord exactly matching Django pattern
- from celery import chord
-
- result = chord(batch_tasks)(
- app.signature(
- "process_batch_callback_api", # Use API-specific callback
- kwargs={
- "execution_id": str(execution_id),
- "pipeline_id": str(pipeline_id) if pipeline_id else None,
- "organization_id": str(schema_name),
- }, # Pass required parameters for API callback
- queue=file_processing_callback_queue,
- )
+ # Route through ``WorkflowOrchestrationUtils.create_chord_execution``
+ # (which delegates to ``CeleryChordBarrier`` under the hood) instead of
+ # calling ``chord(...)`` directly. Same Celery semantics, but the
+ # ``Barrier`` abstraction lets a future Phase 6b swap to
+ # ``RedisDecrBarrier`` behind a flag without touching this call site.
+ # The header tasks + callback now carry the fairness slot for
+ # downstream PG Queue routing — closes the chord-fairness gap noted
+ # in Phase 5.1.
+ result = WorkflowOrchestrationUtils.create_chord_execution(
+ batch_tasks=batch_tasks,
+ callback_task_name="process_batch_callback_api",
+ callback_kwargs={
+ "execution_id": str(execution_id),
+ "pipeline_id": str(pipeline_id) if pipeline_id else None,
+ "organization_id": str(schema_name),
+ },
+ callback_queue=file_processing_callback_queue,
+ app_instance=app,
+ fairness=FairnessKey(
+ org_id=str(schema_name),
+ workload_type=WorkloadType.API,
+ ),
)
if not result:
+ # Two reasons ``result`` can be falsy:
+ # 1. Zero ``batch_tasks`` — the barrier short-circuits and
+ # returns ``None``. Pre-Barrier this would have called
+ # ``chord(empty)(callback)`` which Celery handles by
+ # firing the callback immediately with ``[]``. We
+ # preserve that contract byte-for-byte by dispatching
+ # the callback explicitly with an empty result list, so
+ # workflow_execution_status, API result cache, and
+ # pipeline notifications all run exactly as they would
+ # have pre-Barrier. (Unreachable in practice — upstream
+ # requires non-empty ``hash_values_of_files`` — but
+ # the defence
+ # closes the theoretical gap.)
+ # 2. ``batch_tasks`` is non-empty but the barrier returned
+ # falsy for some other reason — a genuine queue failure.
+ if not batch_tasks:
+ logger.info(
+ f"Execution {execution_id} orchestrated with zero "
+ "batch tasks — firing callback directly with empty "
+ "result list (preserves pre-Barrier zero-files contract)"
+ )
+ # Route through ``queue_backend.dispatch`` (not raw
+ # ``app.send_task``) so the canary in
+ # ``test_dispatch_sites_characterisation.py`` stays
+ # happy and the fairness slot rides on the wire same
+ # as the chord-path callback does. body=[] matches
+ # Celery's chord-with-empty-header semantic exactly.
+ callback_result = dispatch(
+ "process_batch_callback_api",
+ args=[[]],
+ kwargs={
+ "execution_id": str(execution_id),
+ "pipeline_id": str(pipeline_id) if pipeline_id else None,
+ "organization_id": str(schema_name),
+ },
+ queue=file_processing_callback_queue,
+ fairness=FairnessKey(
+ org_id=str(schema_name),
+ workload_type=WorkloadType.API,
+ ),
+ )
+ return {
+ "status": "orchestrated",
+ "execution_id": execution_id,
+ "workflow_id": workflow_id,
+ "task_id": task_id,
+ "files_processed": total_files,
+ "files_from_cache": len(cached_results),
+ "batches_created": 0,
+ "chord_id": callback_result.id,
+ "cached_results": list(cached_results.keys())
+ if cached_results
+ else [],
+ "message": (
+ f"Zero batch tasks for execution {execution_id} — "
+ "callback fired directly with empty result list"
+ ),
+ }
exception = f"Failed to queue execution task {execution_id}"
logger.error(exception)
raise Exception(exception)
diff --git a/workers/general/tasks.py b/workers/general/tasks.py
index aa784c81f..edceacdb7 100644
--- a/workers/general/tasks.py
+++ b/workers/general/tasks.py
@@ -7,7 +7,8 @@ and general workflow executions using internal APIs.
import time
from typing import Any
-from queue_backend import worker_task
+from queue_backend import FairnessKey, worker_task
+from queue_backend.fairness import WorkloadType
from scheduler.tasks import execute_pipeline_task_v2
# Import shared worker infrastructure using new structure
@@ -937,13 +938,20 @@ def _orchestrate_file_processing_general(
# Import to ensure we have the right app context
from worker import app as celery_app
- # Use shared orchestration utility for chord execution
+ # Route through the ``Barrier``-backed helper. The general path
+ # is ETL / non-API workflow execution — ``NON_API`` workload type
+ # so any future PG Queue fairness scheduler can split API traffic
+ # from background workflow processing.
result = WorkflowOrchestrationUtils.create_chord_execution(
batch_tasks=batch_tasks,
callback_task_name=TaskName.PROCESS_BATCH_CALLBACK.value,
callback_kwargs=callback_kwargs,
callback_queue=file_processing_callback_queue,
app_instance=celery_app,
+ fairness=FairnessKey(
+ org_id=organization_id,
+ workload_type=WorkloadType.NON_API,
+ ),
)
if not result:
diff --git a/workers/queue_backend/__init__.py b/workers/queue_backend/__init__.py
index a8a0d681b..2e7395135 100644
--- a/workers/queue_backend/__init__.py
+++ b/workers/queue_backend/__init__.py
@@ -4,8 +4,16 @@ Single place where the substrate choice (Celery today; PG Queue later)
lives. Both entry points are transparent passthroughs to Celery today.
"""
+from .barrier import Barrier, BarrierHandle, CeleryChordBarrier
from .decorator import worker_task
from .dispatch import dispatch
from .fairness import FairnessKey
-__all__ = ["FairnessKey", "dispatch", "worker_task"]
+__all__ = [
+ "Barrier",
+ "BarrierHandle",
+ "CeleryChordBarrier",
+ "FairnessKey",
+ "dispatch",
+ "worker_task",
+]
diff --git a/workers/queue_backend/barrier.py b/workers/queue_backend/barrier.py
new file mode 100644
index 000000000..13c37e520
--- /dev/null
+++ b/workers/queue_backend/barrier.py
@@ -0,0 +1,196 @@
+"""Distributed barrier — abstraction over the chord-completion pattern.
+
+The two production chord call sites (``shared/workflow/execution/
+orchestration_utils.py`` and ``api-deployment/tasks.py``) need to wait
+for a fan-out of header tasks to all complete, then fire a callback
+with the aggregated results. Today that's implemented by Celery's
+``chord(header)(body)`` primitive, which is the highest-risk Celery
+construct at our scale — silent task drops at ~130K-task scale are
+documented in the PG Queue decision journey.
+
+This module provides the ``Barrier`` protocol and a ``CeleryChordBarrier``
+implementation that wraps the existing ``chord(...)`` call 1:1. Lifting
+both call sites to ``Barrier`` gives Phase 6b a place to land an
+alternative implementation (e.g. ``RedisDecrBarrier`` using the labs
+``DECR remaining:{exec_id}`` pattern, or — later in Phase 8 — a
+``PgBarrier`` using SKIP LOCKED) without touching the call sites a
+second time.
+
+**Phase 6a (this PR): wrapper only — behaviour-preserving uplift.**
+``CeleryChordBarrier.enqueue(...)`` produces byte-identical wire output
+to the direct ``chord(...)`` calls it replaces *when ``fairness=None``*.
+When a ``FairnessKey`` is passed (which both production call sites
+now do), each header task and the callback additionally carry an
+``x-fairness-key`` AMQP header. That's an additive wire-level change —
+consumers ignore unknown headers — same posture as Phase 5.1's
+``dispatch()`` plumbing. Mixed-version rolling deploys are safe in
+both directions: old workers ignore the new header; new workers
+handle messages without the header identically to today.
+
+Phase 6b (separate PR) will add ``RedisDecrBarrier`` as a second
+implementation, gated by ``WORKER_BARRIER_BACKEND`` env flag (default
+``chord``). Phase 6c is the rollout.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any, Protocol
+
+from celery import chord
+
+from .fairness import FairnessKey
+from .handle import BarrierHandle
+
+if TYPE_CHECKING:
+ from celery.canvas import Signature
+
+logger = logging.getLogger(__name__)
+
+
+class Barrier(Protocol):
+ """Fan-out-then-callback primitive.
+
+ Semantically: "enqueue these header tasks in parallel; when all
+ complete, fire this callback with the list of header results."
+ The concrete substrate (Celery chord today, Redis DECR or PG
+ SKIP LOCKED later) is hidden behind this protocol.
+ """
+
+ def enqueue(
+ self,
+ header_tasks: list[Signature],
+ *,
+ callback_task_name: str,
+ callback_kwargs: dict[str, Any],
+ callback_queue: str,
+ app_instance: Any,
+ fairness: FairnessKey | None = None,
+ ) -> BarrierHandle | None:
+ """Enqueue ``header_tasks`` and a single callback to fire on completion.
+
+ ``None`` is the **sole signal** that no work was enqueued — it
+ is returned exclusively when ``header_tasks`` is empty. Any
+ substrate-level failure (broker outage, serialisation error,
+ etc.) raises rather than returning ``None``, so callers may
+ treat ``None`` as "no-op, nothing wrong" and a raised
+ exception as a genuine failure. The caller is responsible for
+ guarding ``header_tasks`` (e.g. an early-return when files==0)
+ if a returned ``None`` is semantically distinct from a normal
+ completion in their domain.
+
+ ``fairness``, when provided, is attached as the
+ ``x-fairness-key`` header on every enqueued task and the
+ callback — same wire shape as ``dispatch()`` uses for bare
+ sites. Pass ``None`` for non-workflow-execution callers.
+
+ ``app_instance`` is a Celery app today (the Protocol leaks
+ this concrete dependency for the same reason the
+ ``CeleryChordBarrier`` does — Phase 6b's RedisDecrBarrier will
+ narrow this to a smaller Protocol). The argument is the
+ ``app`` from the producer's task context and must expose
+ ``.signature(name, kwargs=..., queue=..., headers=...)``.
+ """
+ ...
+
+
+class CeleryChordBarrier:
+ """Thin wrapper around ``celery.chord``.
+
+ Behaviour-preserving: the ``chord(header_tasks)(callback_signature)``
+ call is byte-identical to the inline ``chord(...)`` it replaces.
+ The only additional behaviour is the optional ``fairness`` header
+ attachment, which is an additive wire-level change (consumers that
+ don't read the header are unaffected — same backward-compat
+ posture as Phase 5.1's ``dispatch()`` fairness plumbing).
+
+ Future ``RedisDecrBarrier`` and ``PgBarrier`` implementations will
+ live alongside this class and be selected by the
+ ``WORKER_BARRIER_BACKEND`` env flag (Phase 6b).
+ """
+
+ def enqueue(
+ self,
+ header_tasks: list[Signature],
+ *,
+ callback_task_name: str,
+ callback_kwargs: dict[str, Any],
+ callback_queue: str,
+ app_instance: Any,
+ fairness: FairnessKey | None = None,
+ ) -> BarrierHandle | None:
+ """See :class:`Barrier.enqueue`."""
+ # Empty-header guard goes FIRST so a zero-task run skips
+ # signature construction / fairness serialisation entirely.
+ # Any failure in those paths is now constrained to the
+ # non-empty branch where the work is actually needed.
+ if not header_tasks:
+ execution_id = callback_kwargs.get("execution_id")
+ pipeline_id = callback_kwargs.get("pipeline_id")
+ logger.info(
+ f"[exec:{execution_id}] [pipeline:{pipeline_id}] "
+ "Zero header tasks detected — skipping barrier enqueue "
+ "(parent should handle pipeline status updates directly)"
+ )
+ return None
+
+ try:
+ # Use ``fairness.as_header()`` — the single wire-shape
+ # encoder, also used by ``dispatch.py``. Constructing the
+ # header dict inline here would risk silent divergence
+ # from the bare-dispatch path.
+ fairness_headers = fairness.as_header() if fairness else None
+
+ # Build the callback-signature kwargs explicitly. When
+ # ``fairness=None`` the resulting call shape is
+ # byte-identical to the pre-Barrier ``app.signature(name,
+ # kwargs=..., queue=...)`` (no spurious ``headers=None``).
+ signature_kwargs: dict[str, Any] = {
+ "kwargs": callback_kwargs,
+ "queue": callback_queue,
+ }
+ if fairness_headers:
+ signature_kwargs["headers"] = fairness_headers
+ callback_signature = app_instance.signature(
+ callback_task_name, **signature_kwargs
+ )
+
+ # Stamp fairness onto each header signature via
+ # ``Signature.clone().set(headers=...)`` rather than
+ # in-place ``.set(...)`` on the caller's list. Cloning
+ # avoids cross-tenant header leakage if a future retry
+ # path or signature cache ever re-uses the original
+ # ``header_tasks`` list with a different ``FairnessKey``.
+ if fairness_headers:
+ signed_header_tasks = [
+ task.clone().set(headers=fairness_headers) for task in header_tasks
+ ]
+ else:
+ signed_header_tasks = header_tasks
+
+ result = chord(signed_header_tasks)(callback_signature)
+
+ logger.info(
+ f"Barrier enqueued via CeleryChordBarrier — "
+ f"header_tasks={len(signed_header_tasks)}, "
+ f"callback={callback_task_name}, "
+ f"queue={callback_queue}"
+ )
+
+ return result
+
+ except Exception:
+ # ``logger.exception`` auto-attaches the traceback — needed
+ # for debugging broker outages / serialisation failures
+ # at the chord entry point. Add the readily-available
+ # context so a Sentry/log triage gets execution/pipeline/
+ # callback/queue identifiers alongside the stack.
+ execution_id = callback_kwargs.get("execution_id")
+ pipeline_id = callback_kwargs.get("pipeline_id")
+ logger.exception(
+ f"[exec:{execution_id}] [pipeline:{pipeline_id}] "
+ f"Failed to enqueue barrier "
+ f"(callback={callback_task_name}, queue={callback_queue}, "
+ f"header_tasks={len(header_tasks)})"
+ )
+ raise
diff --git a/workers/queue_backend/dispatch.py b/workers/queue_backend/dispatch.py
index 21f0ea68c..cf8d2c274 100644
--- a/workers/queue_backend/dispatch.py
+++ b/workers/queue_backend/dispatch.py
@@ -8,21 +8,12 @@ call sites.
from __future__ import annotations
from collections.abc import Mapping, Sequence
-from typing import Any, Protocol
+from typing import Any
from celery import current_app
from .fairness import FairnessKey
-
-
-class DispatchHandle(Protocol):
- """Minimum contract every dispatch substrate must satisfy.
-
- Celery's ``AsyncResult`` satisfies this via ``.id``; any future
- substrate handle must expose the same attribute.
- """
-
- id: str
+from .handle import DispatchHandle
def dispatch(
diff --git a/workers/queue_backend/handle.py b/workers/queue_backend/handle.py
new file mode 100644
index 000000000..d5b20f5bd
--- /dev/null
+++ b/workers/queue_backend/handle.py
@@ -0,0 +1,37 @@
+"""Shared task-handle protocol.
+
+Single ``TaskHandle`` contract every ``queue_backend`` entry point
+returns: ``dispatch()`` (bare task) and ``Barrier.enqueue()`` (chord /
+future DECR / future PG-SKIP-LOCKED). Celery's ``AsyncResult``
+satisfies this via ``.id``; future non-Celery substrate handles must
+expose the same attribute so call sites that log ``chord_id`` /
+``task_id`` keep working.
+
+Originally split as separate ``DispatchHandle`` and ``BarrierHandle``
+Protocols — they were byte-for-byte identical, which Vishnu's review
+on PR #2024 correctly flagged as a drift risk. Consolidated into
+``TaskHandle`` here; both old names are kept as documentation aliases
+for now.
+"""
+
+from __future__ import annotations
+
+from typing import Protocol
+
+
+class TaskHandle(Protocol):
+ """Return-value contract for any ``queue_backend`` enqueue path.
+
+ Celery's ``AsyncResult`` satisfies this via ``.id``. A future
+ non-Celery substrate handle must expose the same attribute so
+ call sites that log ``chord_id`` / ``task_id`` keep working.
+ """
+
+ id: str
+
+
+# Documentation aliases. Kept so existing imports and docstring
+# references don't break, but they all refer to the same single
+# Protocol — no more drift risk.
+DispatchHandle = TaskHandle
+BarrierHandle = TaskHandle
diff --git a/workers/shared/workflow/execution/orchestration_utils.py b/workers/shared/workflow/execution/orchestration_utils.py
index e70930a1d..6d4cf7ff4 100644
--- a/workers/shared/workflow/execution/orchestration_utils.py
+++ b/workers/shared/workflow/execution/orchestration_utils.py
@@ -7,7 +7,7 @@ chord execution, batch processing, and task coordination utilities.
import os
from typing import Any
-from celery import chord
+from queue_backend import CeleryChordBarrier, FairnessKey
from ...enums import FileDestinationType, PipelineType
from ...enums.worker_enums import QueueName
@@ -15,6 +15,12 @@ from ...infrastructure.logging import WorkerLogger
logger = WorkerLogger.get_logger(__name__)
+# Single ``Barrier`` instance reused across all
+# ``WorkflowOrchestrationUtils.create_chord_execution`` calls. A future
+# Phase 6b will replace this with a factory call (``get_barrier()``)
+# that picks the impl from ``WORKER_BARRIER_BACKEND``.
+_BARRIER = CeleryChordBarrier()
+
class WorkflowOrchestrationUtils:
"""Centralized workflow orchestration patterns and utilities."""
@@ -26,8 +32,16 @@ class WorkflowOrchestrationUtils:
callback_kwargs: dict[str, Any],
callback_queue: str,
app_instance: Any,
+ *,
+ fairness: FairnessKey | None = None,
) -> Any:
- """Standardized chord creation and execution pattern.
+ """Standardized fan-out + callback pattern (Phase 6 ``Barrier``).
+
+ Routes through ``CeleryChordBarrier`` — a thin wrapper around
+ ``celery.chord(header)(body)`` that lets Phase 6b swap the
+ substrate (e.g. to ``RedisDecrBarrier``) without touching this
+ call site a second time. Behaviour is identical to the
+ previous direct ``chord(...)`` call.
Args:
batch_tasks: List of batch task signatures
@@ -35,49 +49,29 @@ class WorkflowOrchestrationUtils:
callback_kwargs: Keyword arguments for callback
callback_queue: Queue name for callback task
app_instance: Celery app instance
+ fairness: Optional ``FairnessKey`` attached as a message
+ header on every header task and the callback — closes
+ the chord-fairness gap that Phase 5.1 (``dispatch()``)
+ deliberately scoped out. See ``queue_backend.fairness``
+ for the slot name constant.
Returns:
- Chord result object or None if no batch tasks
+ Barrier handle (Celery ``AsyncResult``-shaped — ``.id``
+ exposed for chord-id logging) or None if no batch tasks.
Note:
- This consolidates the identical chord creation pattern found in
- api-deployment and general workers.
-
- CRITICAL: Returns None for zero batch tasks, signaling to parent
- that direct pipeline status updates should be handled instead.
+ CRITICAL: Returns None for zero batch tasks, signaling to
+ parent that direct pipeline status updates should be
+ handled instead.
"""
- try:
- callback_signature = app_instance.signature(
- callback_task_name,
- kwargs=callback_kwargs,
- queue=callback_queue,
- )
- # For zero files, skip chord entirely - parent should handle status updates directly
- if not batch_tasks:
- # Extract execution_id from callback kwargs for logging
- execution_id = callback_kwargs.get("execution_id")
- pipeline_id = callback_kwargs.get("pipeline_id")
- logger.info(
- f"[exec:{execution_id}] [pipeline:{pipeline_id}] Zero batch tasks detected - skipping chord execution "
- f"(parent should handle pipeline status updates directly)"
- )
- return None # Signal to parent that no chord was created
-
- # Normal chord execution for non-empty batch tasks
- result = chord(batch_tasks)(callback_signature)
-
- logger.info(
- f"Chord execution started - "
- f"batch_tasks={len(batch_tasks)}, "
- f"callback={callback_task_name}, "
- f"queue={callback_queue}"
- )
-
- return result
-
- except Exception as e:
- logger.error(f"Failed to create chord execution: {e}")
- raise
+ return _BARRIER.enqueue(
+ batch_tasks,
+ callback_task_name=callback_task_name,
+ callback_kwargs=callback_kwargs,
+ callback_queue=callback_queue,
+ app_instance=app_instance,
+ fairness=fairness,
+ )
@staticmethod
def determine_manual_review_routing(
@@ -325,16 +319,33 @@ class WorkflowOrchestrationMixin:
"""Mixin class to add orchestration utilities to worker tasks."""
def create_chord(
- self, batch_tasks, callback_task_name, callback_kwargs, callback_queue
+ self,
+ batch_tasks,
+ callback_task_name,
+ callback_kwargs,
+ callback_queue,
+ *,
+ fairness: FairnessKey | None = None,
):
- """Create chord using standardized pattern."""
+ """Create chord using standardized pattern.
+
+ Forwards ``fairness`` to ``create_chord_execution`` so mixin
+ callers can opt into the chord-fairness plumbing that bare
+ ``dispatch()`` sites get via Phase 5.1. ``None`` keeps the
+ pre-Barrier wire shape exactly.
+ """
# Get app instance from task context
app_instance = getattr(self, "app", None)
if not app_instance:
raise RuntimeError("Celery app instance not available in task context")
return WorkflowOrchestrationUtils.create_chord_execution(
- batch_tasks, callback_task_name, callback_kwargs, callback_queue, app_instance
+ batch_tasks,
+ callback_task_name,
+ callback_kwargs,
+ callback_queue,
+ app_instance,
+ fairness=fairness,
)
def determine_manual_review_routing(self, files, manual_review_config=None):
diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.py
new file mode 100644
index 000000000..a9ae7fa36
--- /dev/null
+++ b/workers/tests/test_barrier.py
@@ -0,0 +1,461 @@
+"""Characterisation tests for the ``Barrier`` abstraction (PG Queue Phase 6a).
+
+The two production chord call sites
+(``WorkflowOrchestrationUtils.create_chord_execution`` and
+``api-deployment/tasks.py``'s inline chord) were lifted behind
+``CeleryChordBarrier`` so a future Phase 6b can swap in
+``RedisDecrBarrier`` (or PG-based) implementations without touching
+call sites.
+
+Three layers of coverage:
+
+1. **Barrier protocol shape** — ``CeleryChordBarrier`` satisfies the
+ ``Barrier`` Protocol and ``AsyncResult`` satisfies ``BarrierHandle``.
+2. **Wire equivalence with the pre-Barrier chord call** — given the
+ same args, ``CeleryChordBarrier.enqueue(...)`` produces the same
+ ``chord(header)(body)`` invocation, the same ``app.signature(...)``
+ args, and the same return value. A regression that bypasses the
+ abstraction or alters the wire surface fails one of these tests.
+3. **Fairness plumbing** — when ``fairness=`` is passed, the fairness
+ header is attached to every enqueued task and to the callback
+ (closes Phase 5.1's chord-fairness gap that was deliberately
+ scoped out).
+
+The single ``chord(...)`` call still lives in
+``queue_backend/barrier.py`` (asserted by the inventory test in
+``test_chord_sites_characterisation.py``).
+"""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from queue_backend import (
+ Barrier,
+ BarrierHandle,
+ CeleryChordBarrier,
+ FairnessKey,
+)
+from queue_backend.fairness import FAIRNESS_HEADER_NAME, WorkloadType
+
+
+def app() -> MagicMock:
+ """Celery-app-shaped mock with a working ``.signature(...)``.
+
+ Used by every test that drives ``CeleryChordBarrier.enqueue(...)``.
+ Shared via fixture rather than a per-class ``_make_app`` helper to
+ keep test classes free of boilerplate (SonarCloud S4144
+ duplication).
+ """
+ mock = MagicMock(name="celery_app")
+ mock.signature.return_value = MagicMock(name="callback_signature")
+ return mock
+
+
+def mock_chord():
+ """Patch ``queue_backend.barrier.chord`` and yield the mock.
+
+ Every wire-equivalence / fairness test patches the same import
+ target — extracting to a fixture removes ~8 identical
+ ``with patch(...) as mock_chord:`` lines (SonarCloud S4144
+ duplication).
+ """
+ with patch("queue_backend.barrier.chord") as m:
+ m.return_value = MagicMock(name="chord_object")
+ yield m
+
+
+# --- Protocol shape ---
+
+
+class TestBarrierProtocolShape:
+ def test_celery_chord_barrier_satisfies_protocol(self):
+ """``CeleryChordBarrier`` is structurally a ``Barrier``.
+
+ Pinned so Phase 6b's ``RedisDecrBarrier`` can be substituted at
+ any call site without runtime ``AttributeError``.
+ """
+ barrier: Barrier = CeleryChordBarrier()
+ assert callable(getattr(barrier, "enqueue", None))
+
+ def test_barrier_handle_protocol_is_minimal(self):
+ """``BarrierHandle`` only requires ``id: str`` — celery's
+ ``AsyncResult`` satisfies this. Future substrate handles must
+ too so existing chord-id logging keeps working."""
+ # MagicMock with .id attribute is structurally a BarrierHandle.
+ handle = MagicMock()
+ handle.id = "task-1"
+ # Treating it as a BarrierHandle at runtime works (Python's
+ # Protocols are duck-typed; this test pins the contract).
+ h: BarrierHandle = handle
+ assert h.id == "task-1"
+
+
+# --- Wire equivalence with the pre-Barrier chord call ---
+
+
+class TestCeleryChordBarrierWireEquivalence:
+ """``CeleryChordBarrier.enqueue(...)`` must produce the same
+ ``chord(batch_tasks)(callback_signature)`` invocation that the
+ original inline ``chord(...)`` calls produced — modulo the
+ additive ``x-fairness-key`` header that the barrier optionally
+ stamps onto signatures.
+ """
+
+ def test_empty_header_returns_none_and_skips_chord(self, app, mock_chord):
+ """Zero-batch contract preserved: barrier returns None,
+ ``chord(...)`` is never called.
+
+ Parent callers (``general/tasks.py``, ``api-deployment/tasks.py``)
+ rely on this signal to handle pipeline status updates directly.
+ """
+ result = CeleryChordBarrier().enqueue(
+ [],
+ callback_task_name="process_batch_callback",
+ callback_kwargs={"execution_id": "exec-1", "pipeline_id": "pipe-1"},
+ callback_queue="file_processing_callback",
+ app_instance=app,
+ )
+
+ assert result is None
+ mock_chord.assert_not_called()
+
+ def test_non_empty_header_invokes_chord_header_then_body(self, app, mock_chord):
+ """The original ``chord(header)(body)`` two-step call must
+ be preserved exactly — a refactor that flattens or reorders
+ these calls would change Celery's chord semantics."""
+ header = [MagicMock(name="h1"), MagicMock(name="h2")]
+
+ CeleryChordBarrier().enqueue(
+ header,
+ callback_task_name="process_batch_callback",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="file_processing_callback",
+ app_instance=app,
+ )
+
+ # Step 1: chord(header)
+ mock_chord.assert_called_once_with(header)
+ # Step 2: chord_obj(callback_signature) — applies the chord
+ mock_chord.return_value.assert_called_once_with(app.signature.return_value)
+
+ def test_callback_signature_args_match_pre_uplift_contract(self, app, mock_chord):
+ """``app.signature(task_name, kwargs=..., queue=...)`` is
+ called with exactly the args the pre-Barrier helper used."""
+ callback_kwargs = {
+ "execution_id": "exec-42",
+ "pipeline_id": "pipe-7",
+ "organization_id": "org-x",
+ }
+
+ CeleryChordBarrier().enqueue(
+ [MagicMock(name="h")],
+ callback_task_name="process_batch_callback_api",
+ callback_kwargs=callback_kwargs,
+ callback_queue="api_file_processing_callback",
+ app_instance=app,
+ )
+
+ # Without ``fairness=``, no ``headers=`` kwarg on the signature
+ # call — matches the pre-Barrier wire byte-for-byte.
+ app.signature.assert_called_once_with(
+ "process_batch_callback_api",
+ kwargs=callback_kwargs,
+ queue="api_file_processing_callback",
+ )
+
+ def test_returns_chord_result_object(self, app, mock_chord):
+ """The barrier handle is whatever ``chord(header)(body)``
+ returns — Celery's ``AsyncResult`` in production. Callers
+ log ``.id`` for chord-id tracing."""
+ chord_result = MagicMock(name="chord_result")
+ mock_chord.return_value.return_value = chord_result
+
+ result = CeleryChordBarrier().enqueue(
+ [MagicMock()],
+ callback_task_name="process_batch_callback",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="file_processing_callback",
+ app_instance=app,
+ )
+
+ assert result is chord_result
+
+ def test_chord_failure_is_re_raised_after_logging(self, app, mock_chord):
+ """If ``chord(...)`` raises (broker outage, serialisation
+ error, etc.), the barrier logs and re-raises — never swallows.
+
+ Without this, callers would treat broker failures as silent
+ zero-task chords and skip pipeline status updates entirely.
+ """
+ mock_chord.side_effect = RuntimeError("broker exploded")
+
+ with pytest.raises(RuntimeError, match="broker exploded"):
+ CeleryChordBarrier().enqueue(
+ [MagicMock()],
+ callback_task_name="process_batch_callback",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="file_processing_callback",
+ app_instance=app,
+ )
+
+
+# --- Fairness plumbing ---
+
+
+class TestCeleryChordBarrierFairnessHeader:
+ """When ``fairness=FairnessKey(...)`` is passed, the barrier
+ stamps ``x-fairness-key`` on every header task and the callback.
+
+ Closes the chord-fairness gap from Phase 5.1: bare ``dispatch()``
+ calls carried fairness; the two chord call sites didn't (they
+ bypassed ``dispatch()`` entirely). After this Phase 6a uplift,
+ every queue-crossing payload on the workflow-execution path now
+ carries the fairness slot.
+ """
+
+ def test_fairness_header_stamped_on_callback_signature(self, app, mock_chord):
+ CeleryChordBarrier().enqueue(
+ [MagicMock(name="h")],
+ callback_task_name="process_batch_callback_api",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="api_file_processing_callback",
+ app_instance=app,
+ fairness=FairnessKey(org_id="org-1", workload_type=WorkloadType.API),
+ )
+
+ headers = app.signature.call_args.kwargs.get("headers")
+ assert headers is not None, (
+ "expected ``headers=`` kwarg on app.signature when fairness "
+ "is passed"
+ )
+ assert headers[FAIRNESS_HEADER_NAME] == {
+ "org_id": "org-1",
+ "workload_type": "api",
+ "pipeline_priority": 5,
+ }
+
+ def test_fairness_header_stamped_on_every_header_task(self, app, mock_chord):
+ """Every batch task in the header carries the fairness header
+ so PG Queue's per-task fairness scheduler can route each
+ independently.
+
+ Header signatures are stamped via ``Signature.clone().set(...)``
+ (not in-place ``.set(...)``) — clone avoids cross-tenant
+ header leakage if a future retry path or signature cache ever
+ re-uses the original ``header_tasks`` list with a different
+ ``FairnessKey``. The test asserts ``.clone().set(headers=...)``
+ is called on each original task.
+ """
+ h1, h2, h3 = (MagicMock(name=f"header_task_{i}") for i in range(3))
+
+ CeleryChordBarrier().enqueue(
+ [h1, h2, h3],
+ callback_task_name="process_batch_callback",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="file_processing_callback",
+ app_instance=app,
+ fairness=FairnessKey(org_id="org-1", workload_type=WorkloadType.NON_API),
+ )
+
+ expected = {
+ FAIRNESS_HEADER_NAME: {
+ "org_id": "org-1",
+ "workload_type": "non_api",
+ "pipeline_priority": 5,
+ }
+ }
+ for task in (h1, h2, h3):
+ # ``.clone()`` produces a fresh signature; the ``.set(...)``
+ # then attaches the fairness header to the clone, leaving
+ # the original ``task`` unchanged.
+ task.clone.assert_called_once_with()
+ task.clone.return_value.set.assert_called_once_with(headers=expected)
+ # Direct ``.set(...)`` on the original is NEVER called —
+ # the whole point of the clone-and-set pattern.
+ task.set.assert_not_called()
+
+ def test_no_fairness_no_header_added(self, app, mock_chord):
+ """When ``fairness=None``, the barrier behaves byte-for-byte
+ like the pre-Barrier chord call — no ``headers=`` kwarg on
+ the signature, no ``.set(headers=...)`` on header tasks.
+
+ Mixed-version rolling deploys are safe because this branch
+ produces the same wire as the old direct chord call.
+ """
+ header = [MagicMock(name="h1"), MagicMock(name="h2")]
+
+ CeleryChordBarrier().enqueue(
+ header,
+ callback_task_name="process_batch_callback",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="file_processing_callback",
+ app_instance=app,
+ )
+
+ # Callback signature: no headers kwarg.
+ assert "headers" not in app.signature.call_args.kwargs
+
+ # Header tasks: ``.set(...)`` not called for headers stamping.
+ for task in header:
+ # ``set`` might be called for other reasons in production,
+ # but never with ``headers=`` when fairness is None.
+ for call in task.set.call_args_list:
+ assert "headers" not in call.kwargs, (
+ "unexpected fairness header stamped without fairness="
+ )
+
+
+# --- Singleton routing (pins the _BARRIER dispatch point) ---
+
+
+class TestOrchestrationUtilsRoutesThroughSingleton:
+ """Pin the ``WorkflowOrchestrationUtils.create_chord_execution``
+ routing to the module-level ``_BARRIER`` singleton.
+
+ A refactor that bypasses the singleton (e.g. inlining
+ ``CeleryChordBarrier().enqueue(...)`` inside the static method,
+ or going back to a direct ``chord(...)`` call) would still pass
+ the inventory canary in ``test_chord_sites_characterisation.py``
+ as long as ``chord(...)`` lives somewhere inside ``barrier.py``.
+ This test closes that gap so the singleton is the unambiguous
+ single dispatch point — ready for Phase 6b's factory swap.
+ """
+
+ def test_create_chord_execution_delegates_to_module_barrier(self):
+ from shared.workflow.execution import orchestration_utils
+ from shared.workflow.execution.orchestration_utils import (
+ WorkflowOrchestrationUtils,
+ )
+
+ app_mock = MagicMock(name="celery_app")
+ app_mock.signature.return_value = MagicMock(name="callback_signature")
+ batch = [MagicMock(name="h1")]
+ fairness = FairnessKey(org_id="org-1", workload_type=WorkloadType.API)
+
+ with patch.object(orchestration_utils, "_BARRIER") as mock_barrier:
+ mock_barrier.enqueue.return_value = MagicMock(name="result")
+ WorkflowOrchestrationUtils.create_chord_execution(
+ batch_tasks=batch,
+ callback_task_name="process_batch_callback_api",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="api_file_processing_callback",
+ app_instance=app_mock,
+ fairness=fairness,
+ )
+
+ mock_barrier.enqueue.assert_called_once_with(
+ batch,
+ callback_task_name="process_batch_callback_api",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="api_file_processing_callback",
+ app_instance=app_mock,
+ fairness=fairness,
+ )
+
+
+# --- Call-site fairness contracts ---
+
+
+class TestCallSiteFairnessContracts:
+ """Pin the ``FairnessKey`` shape each production call site declares.
+
+ A refactor that swaps the workload types, drops the ``fairness=``
+ kwarg, or transposes the org-id source goes undetected by the
+ isolated barrier tests above. These tests assert the contract at
+ the call site boundary.
+ """
+
+ def test_api_deployment_declares_api_workload_with_schema_name(self):
+ """``api-deployment/tasks.py`` must pass ``WorkloadType.API``
+ and ``org_id=str(schema_name)``.
+
+ ``api-deployment`` is not a valid Python identifier (the dash
+ prevents ``import api-deployment.tasks``), so the test reads
+ the source file by path instead of importing the module.
+ """
+ import inspect
+ import pathlib
+
+ # Derive workers root from a sibling importable module
+ # (``queue_backend.barrier``) so the path stays correct under
+ # any test runner working directory.
+ src = inspect.getfile(__import__("queue_backend.barrier", fromlist=["a"]))
+ api_tasks_path = (
+ pathlib.Path(src).parent.parent / "api-deployment" / "tasks.py"
+ )
+ text = api_tasks_path.read_text()
+
+ # Assert the FairnessKey block at the chord call site declares
+ # the contract this PR's commit message claims it does.
+ assert "fairness=FairnessKey(" in text
+ assert "workload_type=WorkloadType.API" in text
+ assert "org_id=str(schema_name)" in text
+
+ def test_general_declares_non_api_workload_with_organization_id(self):
+ """``general/tasks.py`` must pass ``WorkloadType.NON_API`` and
+ ``org_id=organization_id``."""
+ import inspect
+ import pathlib
+
+ src = inspect.getfile(__import__("queue_backend.barrier", fromlist=["a"]))
+ general_tasks_path = (
+ pathlib.Path(src).parent.parent / "general" / "tasks.py"
+ )
+ text = general_tasks_path.read_text()
+
+ assert "fairness=FairnessKey(" in text
+ assert "workload_type=WorkloadType.NON_API" in text
+ assert "org_id=organization_id" in text
+
+
+# --- Zero-files contract (regression pin for T1) ---
+
+
+class TestApiDeploymentZeroFilesContract:
+ """Pin the api-deployment zero-files handler.
+
+ Before the Barrier uplift, ``chord(empty)(callback)`` returned a
+ truthy ``AsyncResult`` (Celery fires the body immediately with
+ ``[]``). The Barrier returns ``None`` for empty headers, so the
+ caller now has to handle the ``None`` case explicitly — otherwise
+ the existing ``if not result: raise`` would map zero-files runs
+ to ``ExecutionStatus.ERROR``.
+
+ The post-Barrier handler explicitly dispatches the callback with
+ an empty result list to preserve pre-Barrier behaviour
+ byte-for-byte: ``workflow_execution_status`` updates, API result
+ caching, and pipeline notifications all run exactly as they would
+ have pre-Barrier. Unreachable in practice (upstream guarantees
+ non-empty ``hash_values_of_files``) but this test pins the defensive
+ contract so a future refactor doesn't silently regress.
+ """
+
+ def test_api_deployment_zero_files_dispatches_callback_with_empty_list(self):
+ import inspect
+ import pathlib
+
+ src = inspect.getfile(__import__("queue_backend.barrier", fromlist=["a"]))
+ api_tasks_path = (
+ pathlib.Path(src).parent.parent / "api-deployment" / "tasks.py"
+ )
+ text = api_tasks_path.read_text()
+
+ # Defensive branch present and dispatches the callback via
+ # the seam (queue_backend.dispatch), not raw send_task — the
+ # dispatch-sites canary in test_dispatch_sites_characterisation.py
+ # enforces that. The literal strings are part of the
+ # externally-observable response + behaviour contract.
+ assert "if not batch_tasks:" in text
+ assert "dispatch(" in text # routed through queue_backend seam
+ assert '"process_batch_callback_api"' in text # callback fired
+ assert "args=[[]]" in text # body=[] matches chord-empty semantic
+ assert '"batches_created": 0' in text
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/workers/tests/test_chord_sites_characterisation.py b/workers/tests/test_chord_sites_characterisation.py
index 7c9d653c2..79a8202e6 100644
--- a/workers/tests/test_chord_sites_characterisation.py
+++ b/workers/tests/test_chord_sites_characterisation.py
@@ -1,21 +1,27 @@
-"""Characterisation tests for the two ``celery.chord`` call sites.
-
-These are the two places in ``workers/`` source that invoke ``chord(...)``
-directly. PR #13 will replace both with a transport-agnostic ``Barrier``
-abstraction matching the labs target architecture's ``DECR remaining:{exec_id}``
-pattern.
-
-This test suite locks down the **current** chord invocation contract so the
-migration can be proved equivalent. Chord is the highest-risk Celery construct
-called out in the PG Queue decision doc (silent task drops at ~130K-task scale)
-— characterising it before refactor is critical.
-
-Sites:
-1. ``shared/workflow/execution/orchestration_utils.py`` — ``WorkflowOrchestrationUtils.create_chord_execution`` (the centralised helper).
-2. ``api-deployment/tasks.py`` — inline chord inside ``_run_workflow_api``.
- Site 2 is exercised only via the inventory test (full characterisation
- requires heavy mocking of the enclosing 273-line function — out of scope
- for a smoke/characterisation pass).
+"""Characterisation tests for the chord-callback boundary, now lifted
+behind the ``Barrier`` abstraction (PG Queue Phase 6a).
+
+Originally written to lock down the inline ``celery.chord(...)`` calls
+at two call sites before they were lifted. Post-uplift the inline
+calls are gone — both call sites delegate to
+``WorkflowOrchestrationUtils.create_chord_execution`` which in turn
+delegates to ``CeleryChordBarrier.enqueue(...)``. The single remaining
+``chord(...)`` call lives inside ``workers/queue_backend/barrier.py``.
+
+Sites covered (assertions unchanged from the original pre-uplift
+suite — they characterise the **same** ``chord(header)(body)``
+contract, now reached one indirection later):
+1. ``shared/workflow/execution/orchestration_utils.py`` —
+ ``WorkflowOrchestrationUtils.create_chord_execution`` (centralised helper).
+2. ``api-deployment/tasks.py`` — same helper, called inline inside
+ ``_run_workflow_api`` (post-uplift it no longer calls ``chord(...)``
+ directly; covered by the inventory test below).
+
+Chord is the highest-risk Celery construct called out in the PG Queue
+decision doc (silent task drops at ~130K-task scale). A future
+Phase 6b will introduce ``RedisDecrBarrier`` as a second implementation
+behind a flag; these tests stay green through that transition because
+they characterise the **Celery chord** code path specifically.
"""
from unittest.mock import MagicMock, patch
@@ -49,9 +55,7 @@ class TestCreateChordExecution:
app = self._make_app_instance()
- with patch(
- "shared.workflow.execution.orchestration_utils.chord"
- ) as mock_chord:
+ with patch("queue_backend.barrier.chord") as mock_chord:
result = WorkflowOrchestrationUtils.create_chord_execution(
batch_tasks=[],
callback_task_name="process_batch_callback",
@@ -72,9 +76,7 @@ class TestCreateChordExecution:
app = self._make_app_instance()
batch_tasks = [MagicMock(name="batch_task_1"), MagicMock(name="batch_task_2")]
- with patch(
- "shared.workflow.execution.orchestration_utils.chord"
- ) as mock_chord:
+ with patch("queue_backend.barrier.chord") as mock_chord:
# chord(batch_tasks) returns a chord object; calling it with the
# callback signature returns the chord result. Both calls must
# happen for this characterisation.
@@ -109,9 +111,7 @@ class TestCreateChordExecution:
"organization_id": "org-x",
}
- with patch(
- "shared.workflow.execution.orchestration_utils.chord"
- ) as mock_chord:
+ with patch("queue_backend.barrier.chord") as mock_chord:
mock_chord.return_value = MagicMock()
WorkflowOrchestrationUtils.create_chord_execution(
batch_tasks=batch_tasks,
@@ -136,9 +136,7 @@ class TestCreateChordExecution:
app = self._make_app_instance()
- with patch(
- "shared.workflow.execution.orchestration_utils.chord"
- ) as mock_chord:
+ with patch("queue_backend.barrier.chord") as mock_chord:
chord_obj = MagicMock()
chord_result = MagicMock(name="chord_result_object")
chord_obj.return_value = chord_result
@@ -162,9 +160,7 @@ class TestCreateChordExecution:
app = self._make_app_instance()
- with patch(
- "shared.workflow.execution.orchestration_utils.chord"
- ) as mock_chord:
+ with patch("queue_backend.barrier.chord") as mock_chord:
mock_chord.side_effect = RuntimeError("broker exploded")
with pytest.raises(RuntimeError, match="broker exploded"):
@@ -215,12 +211,16 @@ class TestWorkflowOrchestrationMixinCreateChord:
callback_queue="file_processing_callback",
)
+ # Mixin now forwards ``fairness=`` (None when caller omits it)
+ # to ``create_chord_execution``. The positional args stay the
+ # same.
mock_static.assert_called_once_with(
batch,
"process_batch_callback",
kwargs,
"file_processing_callback",
task.app,
+ fairness=None,
)
def test_create_chord_raises_when_no_app_bound(self):
@@ -247,14 +247,37 @@ class TestWorkflowOrchestrationMixinCreateChord:
class TestChordSiteInventory:
- """Exactly two chord call sites must exist in workers/ source.
-
- If a third appears, this test fails so PR #13's migration can't silently
- miss it.
+ """Post-Barrier-uplift invariant: exactly ONE ``chord(...)`` call
+ site, inside ``queue_backend/barrier.py``.
+
+ Before the uplift there were two inline ``chord(...)`` calls (in
+ ``orchestration_utils.py`` and ``api-deployment/tasks.py``). The
+ refactor lifted both behind ``CeleryChordBarrier`` so the chord
+ primitive has a single home — a future ``RedisDecrBarrier`` swap
+ can target one location instead of chasing call sites.
+
+ If a third ``chord(...)`` call appears anywhere outside
+ ``barrier.py``, this test fails — preventing a silent regression
+ where a future PR bypasses the abstraction.
"""
- def test_only_two_known_chord_call_sites_in_workers(self):
- """Count chord(...) invocations (not imports) in workers/ source."""
+ _CHORD_HOST = "queue_backend/barrier.py"
+
+ def test_chord_invocation_lives_only_in_barrier(self):
+ """Count chord(...) invocations (not imports) in workers/ source.
+
+ Tightened regex (after the original ``(?:^|[\\s=(])chord\\(``
+ could match docstring prose mentioning ``chord(...)`` after
+ whitespace): now requires assignment form ``= chord(`` (with
+ optional whitespace), which matches the production call at
+ ``barrier.py``'s ``result = chord(header_tasks)(callback_signature)``
+ but rejects docstring / comment mentions like ``the chord(...)
+ primitive``.
+
+ Comment / docstring lines are also explicitly skipped as
+ belt-and-suspenders against a future call site that lacks the
+ ``=`` prefix (e.g. a return-value-discarded ``chord(...)``).
+ """
import pathlib
import re
@@ -264,12 +287,10 @@ class TestChordSiteInventory:
# `workers/shared/tests_helpers/`.
skip_top_dirs = {"tests", "__pycache__", "htmlcov", ".venv"}
- # Match `chord(` as a function call — excludes the bare import line
- # `from celery import chord` and helper method names like `create_chord`.
- # The regex requires `chord` to be preceded by start-of-line / whitespace
- # / `=` / `(` (i.e., a true call expression), not as part of a longer
- # identifier such as `create_chord`.
- pattern = re.compile(r"(?:^|[\s=(])chord\(")
+ # Assignment-form ``= chord(`` — production call sites all
+ # capture the AsyncResult into a variable. Docstring prose
+ # never uses this form.
+ pattern = re.compile(r"=\s*chord\(")
hits = []
for py in workers_root.rglob("*.py"):
@@ -278,21 +299,31 @@ class TestChordSiteInventory:
continue
text = py.read_text()
for line_no, line in enumerate(text.splitlines(), start=1):
+ stripped = line.lstrip()
+ # Skip pure comment lines (belt-and-suspenders — the
+ # regex already rejects most of these via the ``=``
+ # prefix requirement, but a comment like ``# x = chord(...)``
+ # would otherwise sneak through).
+ if stripped.startswith("#"):
+ continue
if pattern.search(line):
hits.append(f"{py.relative_to(workers_root)}:{line_no}")
- # Expected exactly two — in orchestration_utils.py and api-deployment/tasks.py.
- assert len(hits) == 2, (
- f"Expected exactly 2 chord(...) call sites in workers/, found "
- f"{len(hits)}:\n " + "\n ".join(hits)
+ assert len(hits) == 1, (
+ "Expected exactly 1 chord(...) call site in workers/ (inside "
+ f"{self._CHORD_HOST}), found {len(hits)}:\n "
+ + "\n ".join(hits)
+ + "\nPhase 6a lifted both call sites behind CeleryChordBarrier — "
+ "a new direct chord(...) call bypasses the Barrier abstraction "
+ "and breaks the future Phase 6b RedisDecrBarrier swap point."
+ )
+ assert self._CHORD_HOST in hits[0], (
+ f"chord(...) call must live in {self._CHORD_HOST}, found at {hits[0]}"
)
- joined = " ".join(hits)
- assert "shared/workflow/execution/orchestration_utils.py" in joined
- assert "api-deployment/tasks.py" in joined
- def test_chord_import_only_in_two_files(self):
- """`from celery import chord` should appear in exactly the two files
- that actually invoke chord — no other imports lurking."""
+ def test_chord_import_only_in_barrier(self):
+ """`from celery import chord` should appear in exactly one file —
+ ``queue_backend/barrier.py`` — post-Barrier uplift."""
import pathlib
import re
@@ -311,16 +342,13 @@ class TestChordSiteInventory:
if pattern.search(line):
hits.append(f"{py.relative_to(workers_root)}:{line_no}")
- assert len(hits) == 2, (
- f"Expected `from celery import chord` in exactly 2 files, found "
+ assert len(hits) == 1, (
+ f"Expected `from celery import chord` in exactly 1 file, found "
f"{len(hits)}:\n " + "\n ".join(hits)
)
- # Sanity: same files as the call-site canary above. If the imports
- # ever migrate to different files while count stays at 2, this catches
- # it — preventing a silent miss during the Barrier migration.
- joined = " ".join(hits)
- assert "shared/workflow/execution/orchestration_utils.py" in joined
- assert "api-deployment/tasks.py" in joined
+ assert self._CHORD_HOST in hits[0], (
+ f"chord import must live in {self._CHORD_HOST}, found at {hits[0]}"
+ )
if __name__ == "__main__":
diff --git a/workers/tests/test_queue_backend_seam.py b/workers/tests/test_queue_backend_seam.py
index 5a6ac88b5..7075dbaf2 100644
--- a/workers/tests/test_queue_backend_seam.py
+++ b/workers/tests/test_queue_backend_seam.py
@@ -275,7 +275,17 @@ class TestPublicSurface:
def test_all_exports(self):
import queue_backend
- assert set(queue_backend.__all__) == {"FairnessKey", "dispatch", "worker_task"}
+ # Phase 6a added Barrier / BarrierHandle / CeleryChordBarrier.
+ # Phase 6b will add a ``get_barrier`` factory when the
+ # ``WORKER_BARRIER_BACKEND`` flag wiring lands.
+ assert set(queue_backend.__all__) == {
+ "Barrier",
+ "BarrierHandle",
+ "CeleryChordBarrier",
+ "FairnessKey",
+ "dispatch",
+ "worker_task",
+ }
if __name__ == "__main__":