Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3523 [FEAT] PG Queue Phase 6b — RedisDecrBarrier + WORKER_BARRIER_BACKEND flag (#2026)

ali77d09d5a2e09
diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.py
index a56f0eb7c..f4bec94a7 100644
--- a/workers/api-deployment/tasks.py
+++ b/workers/api-deployment/tasks.py
@@ -696,20 +696,11 @@ def _run_workflow_api(
fairness=api_fairness,
)
- if not result:
- # TODO(phase-6b): tighten to ``if result is None:``
- # when ``RedisDecrBarrier`` lands. The ``Barrier`` Protocol
- # declares ``None`` as the *sole* "no work enqueued" signal,
- # so identity-against-None is the principled check. Safe to
- # defer today because Celery's ``AsyncResult`` has no custom
- # ``__bool__`` (always truthy), making the truthiness and
- # identity checks behaviourally equivalent. A future
- # substrate handle with a falsy-but-not-None ``__bool__``
- # would be misrouted into the zero-files path under the
- # current truthiness check — that's the regression this TODO
- # forecloses, alongside the substrate that motivates it.
- #
- # Two reasons ``result`` can be falsy:
+ if result is None:
+ # Identity-against-``None`` mirrors the ``Barrier`` Protocol
+ # contract verbatim: ``None`` is the *sole* signal of "no
+ # work enqueued" (substrate-level failures raise instead).
+ # Two reasons ``result`` can be ``None``:
# 1. Zero ``batch_tasks`` — the barrier short-circuits and
# returns ``None``. Pre-Barrier this would have called
# ``chord(empty)(callback)`` which Celery handles by
diff --git a/workers/queue_backend/__init__.py b/workers/queue_backend/__init__.py
index 2e7395135..3fcb1bab6 100644
--- a/workers/queue_backend/__init__.py
+++ b/workers/queue_backend/__init__.py
@@ -2,18 +2,103 @@
Single place where the substrate choice (Celery today; PG Queue later)
lives. Both entry points are transparent passthroughs to Celery today.
+
+The ``Barrier`` substrate is runtime-selectable via the
+``WORKER_BARRIER_BACKEND`` env var:
+
+- ``chord`` (default) — ``CeleryChordBarrier``, the thin wrapper around
+ ``celery.chord(header)(body)``. Byte-identical wire to the
+ pre-abstraction code path; current production behaviour.
+- ``redis`` — ``RedisDecrBarrier``, the labs-design distributed-counter
+ pattern (``DECR remaining:{exec_id}`` + per-task ``RPUSH``-based
+ result aggregation). Replaces only the chord aggregation primitive;
+ dispatch / retries / monitoring still ride on Celery.
+
+**Default-safety posture.** The env defaults to ``chord``, so a merge
+into ``main`` is zero-impact: a downstream env (``unstract-cloud``,
+self-hosted, etc.) that does not set ``WORKER_BARRIER_BACKEND`` runs
+the same substrate it ran yesterday. The new ``RedisDecrBarrier`` code
+is dormant until an operator explicitly flips the flag.
+
+Unknown values raise — operators get a loud error at module import
+time rather than silently falling back, which would mask a typo'd
+flag in a production env.
"""
+import os
+from enum import StrEnum
+
from .barrier import Barrier, BarrierHandle, CeleryChordBarrier
from .decorator import worker_task
from .dispatch import dispatch
from .fairness import FairnessKey
+from .redis_barrier import (
+ RedisDecrBarrier,
+ barrier_abort,
+ barrier_decr_and_check,
+)
__all__ = [
"Barrier",
+ "BarrierBackend",
"BarrierHandle",
"CeleryChordBarrier",
"FairnessKey",
+ "RedisDecrBarrier",
+ "barrier_abort",
+ "barrier_decr_and_check",
"dispatch",
+ "get_barrier",
"worker_task",
]
+
+
+class BarrierBackend(StrEnum):
+ """Enumeration of ``Barrier`` substrate choices.
+
+ Used as the canonical reference for the ``WORKER_BARRIER_BACKEND``
+ env value, the factory's dispatch table, and tests that flip the
+ env. Subclassing ``StrEnum`` (Python 3.11+) means ``BarrierBackend.CHORD
+ == "chord"`` at runtime, so the enum members can stand in anywhere
+ a string is expected — including ``os.environ`` reads.
+ """
+
+ CHORD = "chord"
+ REDIS = "redis"
+
+
+def get_barrier() -> Barrier:
+ """Return the ``Barrier`` implementation selected by env.
+
+ Reads ``WORKER_BARRIER_BACKEND`` at call time so test harnesses
+ that ``monkeypatch.setenv`` can flip the substrate per-test
+ without a module reload. Production call sites construct the
+ barrier once per process (module-level singleton in
+ ``orchestration_utils.py``) so the env read happens at worker
+ startup — flag flips require pod restart, same posture as every
+ other ``WORKER_*`` env in the codebase.
+
+ Default ``BarrierBackend.CHORD`` mirrors the current production
+ substrate. An unknown value raises ``ValueError`` so a typo'd
+ ``redis`` → ``rediz`` doesn't silently fall back to chord (which
+ would invalidate the canary).
+ """
+ raw = os.getenv("WORKER_BARRIER_BACKEND", BarrierBackend.CHORD.value)
+ try:
+ backend = BarrierBackend(raw.strip().lower())
+ except ValueError:
+ valid = tuple(b.value for b in BarrierBackend)
+ raise ValueError(
+ f"WORKER_BARRIER_BACKEND={raw!r} is not a recognised "
+ f"barrier backend. Valid values: {valid}. Unset the env var "
+ f"to default to {BarrierBackend.CHORD.value!r}."
+ ) from None
+
+ if backend is BarrierBackend.CHORD:
+ return CeleryChordBarrier()
+ if backend is BarrierBackend.REDIS:
+ return RedisDecrBarrier()
+ # Unreachable — StrEnum constructor would have raised above for
+ # anything not in the enum. Defensive raise so the type checker
+ # sees an exhaustive match.
+ raise AssertionError(f"Unhandled BarrierBackend: {backend!r}")
diff --git a/workers/queue_backend/redis_barrier.py b/workers/queue_backend/redis_barrier.py
new file mode 100644
index 000000000..5f79db9cd
--- /dev/null
+++ b/workers/queue_backend/redis_barrier.py
@@ -0,0 +1,803 @@
+"""Redis-DECR Barrier — alternative substrate to ``CeleryChordBarrier``.
+
+Implements the ``Barrier`` Protocol via the labs-design distributed-counter
+pattern (``DECR remaining:{exec_id}``) with per-task result aggregation
+in a Redis list. Selected at runtime by the ``WORKER_BARRIER_BACKEND``
+env flag (default ``chord`` — see ``queue_backend.get_barrier``).
+
+**Why this substrate exists.** Celery's ``chord(header)(body)`` aggregator
+is the highest-risk Celery construct at our scale — documented silent
+task drops at ~130K-task scale (see PG Queue decision journey). This
+substrate replaces *only* the aggregation primitive; everything else
+(dispatch, retries, result backend, monitoring) still rides on Celery.
+The transport is unchanged — only the "wait for N tasks to complete,
+then fire the callback with their results" coordination is moved from
+Celery's chord backend to Redis.
+
+**Wire model.**
+
+1. ``enqueue``: ``SET remaining:{exec_id} N`` with TTL (default 6h,
+ tunable via ``WORKER_BARRIER_KEY_TTL_SECONDS``). The results list
+ is created lazily by the first link task's ``RPUSH`` — its TTL is
+ set inside the Lua script after the push, so both keys share the
+ same expiry window. Each header task is dispatched with
+ ``.link(barrier_decr_and_check.s(...))`` (success) and
+ ``.link_error(barrier_abort.s(...))`` (failure).
+2. Per-task success: the link task runs ``RPUSH results + EXPIRE +
+ DECR remaining`` atomically via a Lua script. If the post-decrement
+ counter reads exactly 0, the link reads the full results list,
+ dispatches the callback with it as the first arg, then deletes
+ the Redis keys.
+3. Per-task failure: ``barrier_abort`` runs as a link_error, DELing
+ both barrier keys + logging the failure. Mirrors Celery chord's
+ default error-propagation semantic (callback isn't invoked when
+ any header fails). The outer task's error handler marks the
+ workflow FAILED; this task only cleans barrier state.
+4. TTL safety net: if the link/link_error never fire for some task
+ (worker crash mid-execution, broker outage), TTL expires the keys
+ after the configured window. Subsequent successful tasks' DECR on
+ the missing counter returns negative; the Lua script's ``< 0``
+ branch DELs cleanly without firing a spurious callback.
+
+**Result-aggregation parity.** The callback receives
+``list[BatchExecutionResult]`` exactly as it does today under
+``CeleryChordBarrier`` — zero callback-side changes. Serialisation
+rides through ``BatchExecutionResult.to_dict()`` / ``from_dict()``
+(the typed callback boundary) ↔ JSON in Redis.
+
+**Atomicity.** The per-task ``RPUSH + DECR`` is atomic via Lua — a
+network partition mid-step can't split-brain the counter and the
+result list. ``DECR`` is also atomic on its own, so exactly one task
+sees the post-decrement 0.
+
+**Mixed-version rolling deploy.** The ``WORKER_BARRIER_BACKEND`` env
+var is read at worker startup, so each pod independently picks its
+substrate. A given execution's tasks all share the publishing pod's
+substrate choice (the barrier is selected once per execution start);
+no cross-substrate aggregation within a single execution.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import logging
+import os
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, TypedDict
+
+from celery.canvas import Signature
+
+from unstract.core.cache.redis_client import create_redis_client
+
+from .decorator import worker_task
+from .fairness import FairnessKey
+from .handle import BarrierHandle
+
+if TYPE_CHECKING:
+ import redis as redis_lib
+
+logger = logging.getLogger(__name__)
+
+
+# Env-var prefix for the barrier's Redis connection. Falls back to
+# ``REDIS_`` (the canonical worker Redis instance) via ``create_redis_client``'s
+# nested-getenv pattern. Operators can point the barrier at a separate
+# Redis by setting ``WORKER_BARRIER_REDIS_HOST`` / ``_PORT`` etc.
+_BARRIER_REDIS_ENV_PREFIX = "WORKER_BARRIER_REDIS_"
+
+# Belt-and-suspenders cleanup for orphaned ``remaining`` / ``results``
+# keys. On the happy path the Lua script ``DEL``s both keys when
+# ``remaining`` reaches 0 — TTL only matters when the counter stalls
+# (e.g. some header tasks fail and the link never fires for them).
+#
+# **Must be strictly > longest plausible execution wall-clock.** If
+# TTL expires while tasks are still pending, the next successful
+# task's ``DECR`` creates the counter at ``-1`` and the script fires
+# the callback with just that task's result — followed by more
+# spurious callbacks as remaining tasks complete.
+#
+# Reference budget: the operator-tuned ceiling is whatever
+# ``FILE_PROCESSING_TASK_TIME_LIMIT`` is set to in the deployment env
+# (workers/sample.env ships 7200/2h; docker/sample.env up to 10800/3h).
+# ``process_file_batch`` has ``max_retries=0`` so no retry amplification —
+# worst case per batch is the time-limit ceiling. Multi-batch
+# executions ≈ max(per-batch time) since batches run in parallel
+# (worker concurrency is the limiter, not serial wait).
+#
+# 6h default gives 2-3x margin over the documented per-batch ceiling
+# while staying in the same order of magnitude as
+# ``FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000`` (5h). Operators with
+# longer workflows (e.g. multi-step pipelines with chained barriers)
+# or shorter known max-execution-time should tune via
+# ``WORKER_BARRIER_KEY_TTL_SECONDS``.
+_KEY_TTL_DEFAULT_SECONDS = 6 * 60 * 60 # 6h
+
+
+def _key_ttl_seconds() -> int:
+ """Read the TTL from env, applying the default only on absence.
+
+ Read at call time (not module import) so a test
+ ``monkeypatch.setenv`` flips the value without a module reload.
+
+ Invalid values (non-int, negative, zero) **raise**, matching the
+ posture in ``get_barrier()`` where ``WORKER_BARRIER_BACKEND=rediz``
+ raises rather than silently falling back to chord. A misconfigured
+ TTL shorter than execution wall-clock is a correctness issue per
+ this file's docstring (would cause spurious callback fires) —
+ operators get the same loud-on-misconfig signal as for the
+ backend flag.
+ """
+ raw = os.getenv("WORKER_BARRIER_KEY_TTL_SECONDS")
+ if raw is None:
+ return _KEY_TTL_DEFAULT_SECONDS
+ try:
+ value = int(raw)
+ except ValueError as e:
+ raise ValueError(
+ f"WORKER_BARRIER_KEY_TTL_SECONDS={raw!r} is not an integer. "
+ f"Unset the env var to default to {_KEY_TTL_DEFAULT_SECONDS}s "
+ f"(6h)."
+ ) from e
+ if value <= 0:
+ raise ValueError(
+ f"WORKER_BARRIER_KEY_TTL_SECONDS={value} must be a positive "
+ f"integer. Unset the env var to default to "
+ f"{_KEY_TTL_DEFAULT_SECONDS}s (6h)."
+ )
+ return value
+
+
+# Atomic ``RPUSH + EXPIRE + DECR``: returns ``(remaining, results)``.
+#
+# Three branches the Python side reads off the returned counter:
+#
+# remaining > 0 — pending, no results returned (still aggregating)
+# remaining == 0 — complete, all results returned (we are the last).
+# Keys are NOT DEL'd here — Python defers the DEL
+# until after ``apply_async()`` succeeds, so that
+# a callback-dispatch failure leaves the keys (and
+# their TTL) in place rather than stranding the
+# execution with no state + no recovery path.
+# remaining < 0 — TTL-expired counter (or replay after cleanup);
+# keys DEL'd inside the script to prevent further
+# spurious fires, no callback dispatched.
+#
+# Setting the ``results`` key TTL inside the script — ``EXPIRE`` on
+# the call AFTER ``RPUSH`` ensures the key exists when EXPIRE runs
+# (issuing EXPIRE before any push is a no-op on a non-existent key,
+# which is the bug the previous version had). Repeated EXPIRE per
+# RPUSH is cheap and keeps both keys' TTLs aligned.
+_RPUSH_DECR_LUA = """
+local remaining_key = KEYS[1]
+local results_key = KEYS[2]
+local result_json = ARGV[1]
+local ttl_seconds = tonumber(ARGV[2])
+redis.call("RPUSH", results_key, result_json)
+redis.call("EXPIRE", results_key, ttl_seconds)
+local remaining = redis.call("DECR", remaining_key)
+if remaining == 0 then
+ -- We're the last task. LRANGE the results for the Python side to
+ -- dispatch the callback. DO NOT DEL the keys here — Python defers
+ -- the DEL until after apply_async() succeeds so a dispatch failure
+ -- (broker outage, serialisation error) leaves the keys + TTL in
+ -- place. Without this deferral, an apply_async failure would
+ -- strand the execution with both keys gone, no TTL, no Celery
+ -- retry (max_retries=0), and no link_error fallback — strictly
+ -- worse than the chord baseline (where Celery's chord backend
+ -- owns + retries body invocation).
+ local all_results = redis.call("LRANGE", results_key, 0, -1)
+ return {remaining, all_results}
+end
+if remaining < 0 then
+ -- Counter was TTL-expired (or already cleaned up) when this task
+ -- ran. ``DECR`` on a missing key created it at ``-1``; subsequent
+ -- tasks would land here too. DEL both keys so neither this task
+ -- nor any subsequent one fires a spurious callback; the abandoned
+ -- branch on the Python side surfaces an error-severity log.
+ redis.call("DEL", remaining_key, results_key)
+ return {remaining, {}}
+end
+return {remaining, {}}
+"""
+
+
+# Process-local cached Redis client. Lazy-initialised on first call
+# within a worker process; reused across every ``barrier_decr_and_check``
+# / ``barrier_abort`` / ``RedisDecrBarrier`` invocation in that process.
+#
+# Why a singleton: ``create_redis_client`` builds a fresh
+# ``ConnectionPool`` each call. Without caching, a 1000-task fan-out
+# would spin up 1000 separate pools (1000 TCP handshakes torn down in
+# sequence) — exactly the scale this substrate exists to handle.
+#
+# Fork safety: Celery's prefork model re-imports this module in each
+# child process, so each worker process gets its own fresh
+# ``_redis_client_singleton`` — no socket sharing across fork
+# boundaries (which would corrupt state).
+#
+# Thread safety: redis-py's ``ConnectionPool`` is internally thread-safe,
+# so concurrent users of the cached client are fine. The lazy-init
+# itself is not locked — two threads racing on the first call could
+# briefly create two pools, with one being orphaned. The cost is one
+# short-lived pool (immediately GC'd), not a correctness issue.
+_redis_client_singleton: redis_lib.Redis | None = None
+
+
+def _get_redis_client() -> redis_lib.Redis:
+ """Return a process-cached Redis client for the barrier.
+
+ ``create_redis_client``'s nested-getenv fallback only covers
+ HOST/PORT/PASSWORD/USER/DB — it does NOT cross-fall-back
+ SENTINEL_MODE or SSL. In a deployment where the canonical Redis
+ is Sentinel-backed or TLS-secured, leaving ``WORKER_BARRIER_REDIS_*``
+ unset and relying on the documented HOST fallback would result in
+ the barrier inheriting the right host/port but connecting
+ standalone/plaintext — which fails (or worse, connects without
+ TLS) every execution the moment the flag flips.
+
+ To avoid that footgun: when no barrier-specific HOST is set, use
+ the canonical ``REDIS_`` prefix *directly* so we inherit the FULL
+ canonical config (including Sentinel + SSL). When an operator
+ has set ``WORKER_BARRIER_REDIS_HOST`` they've opted into a
+ dedicated barrier Redis and are responsible for the full config
+ (HOST + PORT + ... + SENTINEL_MODE + SSL).
+
+ ``decode_responses=True`` so ``LRANGE`` returns ``list[str]``
+ (we JSON-decode each entry).
+ """
+ global _redis_client_singleton
+ if _redis_client_singleton is None:
+ # If no dedicated barrier-Redis host is configured, use the
+ # canonical REDIS_ prefix so Sentinel/SSL config inherits too.
+ if os.getenv(f"{_BARRIER_REDIS_ENV_PREFIX}HOST"):
+ env_prefix = _BARRIER_REDIS_ENV_PREFIX
+ else:
+ env_prefix = "REDIS_"
+ _redis_client_singleton = create_redis_client(
+ env_prefix=env_prefix,
+ decode_responses=True,
+ socket_timeout=5,
+ socket_connect_timeout=5,
+ )
+ return _redis_client_singleton
+
+
+def _remaining_key(execution_id: str) -> str:
+ return f"barrier:remaining:{execution_id}"
+
+
+def _results_key(execution_id: str) -> str:
+ return f"barrier:results:{execution_id}"
+
+
+def _abort_lock_key(execution_id: str) -> str:
+ """Per-execution dedup lock for ``barrier_abort``.
+
+ Set via ``SET NX EX`` in ``barrier_abort`` to collapse N concurrent
+ header-task failures into a single Sentry/error event. MUST be
+ cleared at the start of a new barrier (in ``enqueue``) — otherwise
+ a retry that reuses an ``execution_id`` would hit a stale lock,
+ early-exit the abort path, and silently mask the failed retry as
+ successful.
+ """
+ return f"barrier:abort_lock:{execution_id}"
+
+
+class RedisDecrBarrier:
+ """``Barrier`` implementation via Redis ``DECR remaining`` pattern.
+
+ Drop-in replacement for ``CeleryChordBarrier`` from the call
+ sites' perspective — same ``enqueue(header_tasks, ...)`` signature,
+ same ``BarrierHandle | None`` return contract, same fairness
+ plumbing. The substrate difference is invisible above the
+ ``Barrier`` Protocol.
+
+ See module docstring for the wire model, atomicity, and failure
+ semantics.
+ """
+
+ def __init__(self, redis_client: redis_lib.Redis | None = None) -> None:
+ """Args:
+ redis_client: Optional pre-built client. Tests pass a fake
+ (``fakeredis``) here. Production leaves it ``None`` so
+ the barrier builds its own from env on first use.
+ """
+ self._redis_client = redis_client
+
+ @property
+ def redis(self) -> redis_lib.Redis:
+ """Lazy-init the Redis client.
+
+ Built on first use rather than at construction time so the
+ module can be imported (and the worker task registered) in
+ contexts where Redis isn't yet reachable (e.g. test collection,
+ worker boot before networking up).
+
+ Production path: ``self._redis_client is None`` → falls back to
+ (and caches) the module-level ``_redis_client_singleton``, so
+ the instance and the standalone ``barrier_decr_and_check`` /
+ ``barrier_abort`` worker tasks all share the same
+ ``ConnectionPool``.
+
+ Test path: ``__init__(redis_client=...)`` bypasses this branch
+ entirely — fixtures inject a ``MagicMock`` / ``fakeredis``
+ without touching the module-level singleton (which would leak
+ across tests).
+ """
+ if self._redis_client is None:
+ self._redis_client = _get_redis_client()
+ return self._redis_client
+
+ def enqueue(
+ self,
+ header_tasks: list[Signature],
+ *,
+ callback_task_name: str,
+ callback_kwargs: dict[str, Any],
+ callback_queue: str,
+ app_instance: Any,
+ fairness: FairnessKey | None = None,
+ ) -> BarrierHandle | None:
+ """See :class:`queue_backend.barrier.Barrier.enqueue`.
+
+ Mirrors ``CeleryChordBarrier.enqueue`` semantics:
+
+ - Empty ``header_tasks`` → ``None`` (caller handles the
+ zero-files contract); no Redis keys touched.
+ - Any substrate-level failure (Redis down, dispatch error,
+ serialisation issue) → raises.
+ - Success → returns a ``BarrierHandle`` whose ``.id`` is the
+ execution id (no ``AsyncResult`` since there's no Celery
+ chord wrapping the group; the id is what the call site
+ logs for chord-id traceability).
+
+ Note: ``app_instance`` is accepted for ``Barrier`` Protocol
+ parity with ``CeleryChordBarrier`` but unused by this
+ substrate. The callback signature is built inside the link
+ task via ``celery.current_app.signature(...)`` (so the link
+ runs against the *worker's* app, not the producer's). Keeping
+ the parameter in the signature lets call sites swap between
+ substrates without per-substrate adapter code.
+ """
+ # Explicit ``del`` documents the Protocol-parity intent and
+ # satisfies static linters' unused-parameter check.
+ del app_instance
+ if not header_tasks:
+ execution_id = callback_kwargs.get("execution_id")
+ pipeline_id = callback_kwargs.get("pipeline_id")
+ logger.info(
+ f"[exec:{execution_id}] [pipeline:{pipeline_id}] "
+ "Zero header tasks detected — skipping barrier enqueue "
+ "(parent should handle pipeline status updates directly)"
+ )
+ return None
+
+ execution_id = callback_kwargs.get("execution_id")
+ if not execution_id:
+ raise ValueError(
+ "RedisDecrBarrier requires execution_id in callback_kwargs — "
+ "it's the key suffix for the per-execution remaining/results "
+ "Redis keys"
+ )
+ execution_id = str(execution_id)
+
+ try:
+ fairness_headers = fairness.as_header() if fairness else None
+
+ # The link task body needs to know:
+ # - which execution's counter to decrement
+ # - how to reconstruct + dispatch the callback when count hits 0
+ # All Celery-serialisable so the link can run on any worker.
+ callback_descriptor: CallbackDescriptor = {
+ "task_name": callback_task_name,
+ "kwargs": callback_kwargs,
+ "queue": callback_queue,
+ "fairness_headers": fairness_headers,
+ }
+
+ # Initialise the counter before any header task can fire.
+ # ``SET remaining N EX <ttl>`` overwrites any stale counter
+ # left from a prior run with the same exec_id; the upfront
+ # ``DEL results, abort_lock`` (below) clears the rest of
+ # the prior-run state (the results list grows via RPUSH so
+ # we must clear it explicitly; the abort_lock would otherwise
+ # silently mask a retry's failure as success — see the
+ # comment block at the DELETE call). ``ttl`` is
+ # ``_key_ttl_seconds()`` (6h default).
+ ttl_seconds = _key_ttl_seconds()
+ # Clear any leftover state from a prior execution with this
+ # execution_id (including the dedup lock written by a
+ # previous run's ``barrier_abort``). Without DELing the
+ # abort_lock here, a retry that reuses execution_id would
+ # find the stale lock and route every ``barrier_abort``
+ # straight to the deduplicated branch — leaving B's
+ # ``remaining``/``results`` keys uncleaned. In-flight
+ # successful tasks from B would then hit ``remaining == 0``
+ # via DECR and fire the callback with partial results,
+ # silently masking the failed retry as successful.
+ self.redis.delete(
+ _results_key(execution_id),
+ _abort_lock_key(execution_id),
+ )
+ self.redis.set(
+ _remaining_key(execution_id),
+ len(header_tasks),
+ ex=ttl_seconds,
+ )
+ # ``results`` key TTL is set inside the Lua script on
+ # first RPUSH — ``EXPIRE`` on a non-existent key is a
+ # no-op, so we can't pre-set it here.
+
+ # Stamp fairness on each header task and attach the link
+ # (success) + link_error (failure) callbacks.
+ # ``Signature.clone()`` avoids mutating the caller's list
+ # (same reasoning as ``CeleryChordBarrier``).
+ link_signature = barrier_decr_and_check.s(
+ execution_id=execution_id,
+ callback_descriptor=callback_descriptor,
+ )
+ # link_error explicitly propagates header-task failures —
+ # mirrors Celery chord's default error semantic (chord
+ # callback isn't invoked on header failure). Without this,
+ # a failed header task would leave the counter stuck and
+ # the execution would hang until TTL cleanup. The abort
+ # task DELs the barrier keys + logs the error; the outer
+ # task's error path is responsible for the workflow
+ # status update.
+ link_error_signature = barrier_abort.s(execution_id=execution_id)
+ for i, task in enumerate(header_tasks):
+ try:
+ cloned = task.clone()
+ if fairness_headers:
+ cloned.set(headers=fairness_headers)
+ cloned.link(link_signature)
+ cloned.link_error(link_error_signature)
+ cloned.apply_async()
+ except Exception:
+ # Mid-loop dispatch failure: tasks 0..i-1 are
+ # already in flight with the link attached, but
+ # ``i`` of N never made it to the broker. The
+ # counter would never reach 0 → execution hangs.
+ # DEL the barrier keys so the in-flight links'
+ # DECR-on-missing returns negative and the Lua
+ # script's ``< 0`` branch DELs cleanly. Re-raise
+ # so the caller's status-update path marks the
+ # workflow ERROR.
+ self.redis.delete(
+ _remaining_key(execution_id),
+ _results_key(execution_id),
+ )
+ # ``logger.exception`` captures the underlying
+ # apply_async failure cause (broker timeout vs
+ # serialisation error vs routing error) alongside
+ # the orphan-count context — without it, a reader
+ # of this log can't tell why dispatch failed and
+ # has to cross-reference the outer handler's log.
+ logger.exception(
+ f"[exec:{execution_id}] apply_async failed at "
+ f"task {i}/{len(header_tasks)}; {i} orphan tasks "
+ f"already dispatched. Barrier keys DEL'd to "
+ f"prevent spurious callback fires from the "
+ f"orphan tasks' link decrements."
+ )
+ raise
+
+ logger.info(
+ f"Barrier enqueued via RedisDecrBarrier — "
+ f"exec_id={execution_id}, "
+ f"header_tasks={len(header_tasks)}, "
+ f"callback={callback_task_name}, "
+ f"queue={callback_queue}"
+ )
+
+ # Handle's ``.id`` is the execution id — same call sites
+ # log ``chord_id`` from this; the semantic shifts from
+ # "Celery chord aggregator task id" to "execution id" but
+ # both are stable per-execution identifiers and the log
+ # consumers don't depend on the underlying task vs exec id
+ # distinction.
+ return _RedisBarrierHandle(id=execution_id)
+
+ except Exception:
+ execution_id = callback_kwargs.get("execution_id")
+ pipeline_id = callback_kwargs.get("pipeline_id")
+ logger.exception(
+ f"[exec:{execution_id}] [pipeline:{pipeline_id}] "
+ f"Failed to enqueue barrier via Redis "
+ f"(callback={callback_task_name}, queue={callback_queue}, "
+ f"header_tasks={len(header_tasks)})"
+ )
+ raise
+
+
+class CallbackDescriptor(TypedDict):
+ """Shape of the dict baked into the link signature and re-read on
+ the worker that runs ``barrier_decr_and_check``.
+
+ This descriptor crosses a serialisation boundary (Celery
+ ``signature(...)`` → broker → consumer worker), so the four-key
+ contract is otherwise enforced only by string literals duplicated
+ across producer and consumer. Typing it as a ``TypedDict`` gives
+ the type checker a chance to catch typos / renames before they
+ surface as remote ``KeyError`` mid-aggregation.
+
+ ``fairness_headers`` is always present in the dict; ``None`` when
+ the producer passed no ``FairnessKey``.
+ """
+
+ task_name: str
+ kwargs: dict[str, Any]
+ queue: str
+ fairness_headers: dict[str, Any] | None
+
+
+@dataclass(frozen=True, slots=True)
+class _RedisBarrierHandle:
+ """Minimal ``BarrierHandle`` implementation.
+
+ Just an ``id`` attribute (the execution id) so call sites that
+ log ``handle.id`` for chord-id tracing keep working. A class
+ (rather than a ``namedtuple``) gives a clear home for the
+ docstring + ``slots=True`` memory control; the ``BarrierHandle``
+ Protocol is structural so any object exposing ``id: str`` would
+ type-check.
+ """
+
+ id: str
+
+
+@worker_task(
+ name="barrier_decr_and_check",
+ # ``max_retries=0`` — the link task is intentionally non-idempotent
+ # at the Redis-script level (RPUSH + DECR are committed atomically,
+ # but a Celery retry would replay the script and double-count). Any
+ # Celery-level retry would cause a second RPUSH + DECR, corrupting
+ # the aggregation. If the link fails after the Lua commit (e.g. the
+ # subsequent callback ``apply_async`` raises), the counter has
+ # already advanced — TTL-driven cleanup is the safety net, and the
+ # outer execution's error path marks the workflow status.
+ #
+ # Trade-off: a transient Redis blip on the link's Lua call means
+ # this task's contribution is lost (counter doesn't advance) and
+ # the execution hangs until TTL. Acceptable because (a) the
+ # alternative — letting Celery retry — corrupts the aggregation,
+ # and (b) Redis blips are operator-visible via monitoring, unlike
+ # silent double-counts.
+ max_retries=0,
+)
+def barrier_decr_and_check(
+ result: Any,
+ *,
+ execution_id: str,
+ callback_descriptor: CallbackDescriptor,
+) -> dict[str, Any]:
+ """Per-task link callback for ``RedisDecrBarrier``.
+
+ Runs after each header task completes successfully. Pushes the
+ task's result into the per-execution Redis list, decrements the
+ counter, and (if the post-decrement counter reads 0) dispatches
+ the aggregating callback with the full result list.
+
+ ``result`` is whatever the header task returned — for ``process_file_batch``
+ that's a ``BatchExecutionResult`` already serialised to a dict via
+ the typed callback boundary. We pass it through ``json.dumps`` to
+ survive the Redis round-trip.
+ """
+ from celery import current_app
+
+ try:
+ # No ``default=str`` — that would silently stringify any non-
+ # JSON-safe leaf (datetime, UUID, Decimal, sets...) and mask
+ # the typed-boundary regression the surrounding ``try/except``
+ # is here to surface. The header task's return value is
+ # expected to be a fully JSON-safe dict per the
+ # ``BatchExecutionResult.to_dict()`` contract; anything else
+ # should fail loudly here so the regression surfaces.
+ result_json = json.dumps(result)
+ except (TypeError, ValueError):
+ logger.exception(
+ f"[exec:{execution_id}] Header task result is not "
+ f"JSON-serialisable — barrier aggregation cannot proceed. "
+ f"This indicates a typed-boundary regression; the "
+ f"BatchExecutionResult.to_dict() contract must produce a "
+ f"JSON-safe shape."
+ )
+ raise
+
+ redis_client = _get_redis_client()
+ script = redis_client.register_script(_RPUSH_DECR_LUA)
+ remaining, raw_results = script(
+ keys=[
+ _remaining_key(execution_id),
+ _results_key(execution_id),
+ ],
+ args=[result_json, _key_ttl_seconds()],
+ )
+
+ logger.info(f"[exec:{execution_id}] Barrier link DECR → remaining={remaining}")
+
+ if remaining > 0:
+ # More tasks pending — nothing to do.
+ return {"status": "pending", "remaining": remaining}
+
+ if remaining < 0:
+ # TTL-expired counter (or replay after cleanup). The Lua
+ # script already DEL'd both keys to prevent further spurious
+ # fires on subsequent task completions. No callback dispatch.
+ #
+ # Logged at ERROR (not WARNING) because reaching this branch
+ # means the barrier was torn down out from under in-flight
+ # tasks — by definition an abnormal terminal state where the
+ # execution almost certainly did not complete normally.
+ # Returning normally from a .link is indistinguishable from
+ # success to Celery; the ERROR log is the only execution-id-
+ # tagged signal an operator can correlate to the hung/
+ # incorrectly-statused execution.
+ logger.error(
+ f"[exec:{execution_id}] Barrier abandoned — counter went "
+ f"negative (remaining={remaining}); keys were TTL-expired "
+ f"or torn down. No callback dispatched; execution likely "
+ f"in an inconsistent terminal state and needs investigation."
+ )
+ return {"status": "abandoned", "remaining": remaining}
+
+ # remaining == 0: we're the last task. Dispatch the callback.
+ all_results = [json.loads(r) for r in raw_results]
+ callback_task_name = callback_descriptor["task_name"]
+ callback_kwargs = callback_descriptor["kwargs"]
+ callback_queue = callback_descriptor["queue"]
+ fairness_headers = callback_descriptor.get("fairness_headers")
+
+ callback_signature = current_app.signature(
+ callback_task_name,
+ args=[all_results],
+ kwargs=callback_kwargs,
+ queue=callback_queue,
+ **({"headers": fairness_headers} if fairness_headers else {}),
+ )
+ # Dispatch FIRST; DEL the barrier keys only after dispatch succeeds.
+ #
+ # If apply_async raises (broker outage, serialisation error,
+ # routing failure), the exception propagates. The barrier keys
+ # are NOT yet DEL'd, so:
+ # - Subsequent late-arriving link tasks (if any) hit the Lua
+ # ``< 0`` branch (counter is still 0 — DECR-on-zero would
+ # yield -1) and exit cleanly via the abandoned branch.
+ # - The keys TTL-expire eventually — Redis cleans them up.
+ # - The link task's failure surfaces via Celery's standard
+ # task-failure channels.
+ #
+ # If we DEL'd before apply_async (the previous design), an
+ # apply_async failure would strand the execution with no keys,
+ # no TTL, no Celery retry (max_retries=0), and no link_error
+ # fallback — silent infinite hang. The deferred DEL is what
+ # restores parity-or-better with the chord baseline.
+ callback_result = callback_signature.apply_async()
+ redis_client.delete(
+ _remaining_key(execution_id),
+ _results_key(execution_id),
+ )
+
+ logger.info(
+ f"[exec:{execution_id}] Barrier complete — "
+ f"fired callback {callback_task_name} on {callback_queue} "
+ f"with {len(all_results)} aggregated results "
+ f"(callback_task_id={callback_result.id})"
+ )
+
+ return {
+ "status": "complete",
+ "callback_task_id": callback_result.id,
+ "aggregated_count": len(all_results),
+ }
+
+
+@worker_task(
+ name="barrier_abort",
+ # Same posture as ``barrier_decr_and_check`` — non-idempotent at
+ # the Redis level (DEL is fine to repeat, but we don't want
+ # double-logged abort events polluting Sentry).
+ max_retries=0,
+)
+def barrier_abort(
+ request: Any = None,
+ exc: Any = None,
+ traceback: Any = None,
+ *,
+ execution_id: str,
+) -> dict[str, Any]:
+ """``link_error`` callback: header task failed → clean up barrier state.
+
+ Celery 5.5 invokes new-style errbacks as
+ ``errback(request, exc, traceback)`` only for ``bind=False`` tasks
+ with arity > 1. If a worker hits the ``NotRegistered`` fallback
+ during a mixed-version rolling deploy (or if this task is ever
+ switched to ``bind=True``), Celery calls the errback old-style
+ with just ``(task_id,)``. The defaults on ``request`` / ``exc`` /
+ ``traceback`` mean the old-style path degrades to a clean
+ ``exc=None`` log line rather than a confusing
+ ``TypeError: missing required positional arguments``.
+
+ Mirrors Celery chord's default error semantic: when any header
+ task fails, the chord callback isn't invoked. Without explicit
+ cleanup the ``remaining`` counter would stay above 0 until TTL
+ expiry, the ``results`` key would leak, and any subsequent
+ successful header tasks' link would still DECR — eventually
+ spurious-firing the callback with partial results (or via the
+ Lua ``< 0`` branch, abandoning the execution but only after the
+ extra task work).
+
+ We DEL the barrier keys here so the in-flight successful tasks'
+ link DECR returns negative → Lua ``< 0`` branch fires →
+ aggregation cleanly abandoned. **Terminal status drivers**: this
+ task does NOT mark the workflow FAILED — the outer orchestrators
+ (``_run_workflow_api`` / ``_orchestrate_file_processing_general``)
+ wrap only the *synchronous* fan-out in try/except and have
+ already returned by the time ``barrier_abort`` runs. Workflow
+ terminal status on async header failure is driven by per-file
+ updates inside ``process_file_batch`` plus the eventual
+ aggregating callback (which won't fire because we DEL'd the
+ keys). A future Phase 6b' refinement may have ``barrier_abort``
+ itself drive the execution-level status update.
+
+ **Concurrent-failure dedup.** Every header task attaches the same
+ ``link_error``, so N simultaneous failures would otherwise fire N
+ ``barrier_abort`` executions — each calling ``logger.error`` and
+ each producing its own Sentry event for what is a *single* logical
+ execution failure. A ``SET NX`` lock on
+ ``barrier:abort_lock:{execution_id}`` makes the first abort win;
+ subsequent aborts for the same execution see the lock present and
+ early-exit silently. The DELs are idempotent anyway (DEL on a
+ missing key is a no-op), but the dedup collapses the alert noise.
+
+ **Lock release on DELETE failure.** If SET NX succeeds (this
+ abort wins) but the subsequent DELETE then raises (mid-abort
+ Redis blip), the lock stays held while the barrier keys are NOT
+ deleted. Every sibling ``barrier_abort`` would then hit the
+ dedup early-exit and do nothing, leaving the counter alive for
+ in-flight successful tasks to DECR to 0 → callback fires with
+ partial results → silent success masking. To close this hole,
+ we explicitly release the abort_lock if the DELETE raises, so
+ a sibling abort can re-acquire it and complete the cleanup.
+ """
+ redis_client = _get_redis_client()
+ # ``SET NX EX`` — first-write-wins with auto-expiry. TTL matches
+ # the barrier keys' TTL so the lock cleans up alongside them.
+ # The lock is also explicitly DEL'd by ``enqueue`` at the start
+ # of any new execution with the same execution_id, to prevent a
+ # stale lock from masking a retry's failure as success.
+ if not redis_client.set(
+ _abort_lock_key(execution_id), "1", ex=_key_ttl_seconds(), nx=True
+ ):
+ # Another barrier_abort for this execution already won the
+ # SET NX race. Silently exit — no log, no Sentry, no
+ # duplicate DEL.
+ return {
+ "status": "deduplicated",
+ "execution_id": execution_id,
+ }
+ try:
+ redis_client.delete(
+ _remaining_key(execution_id),
+ _results_key(execution_id),
+ )
+ except Exception:
+ # DELETE failed AFTER we won the SET NX. Release the lock so
+ # subsequent aborts aren't dedup-suppressed into no-ops while
+ # the barrier keys still exist. ``suppress`` because if the
+ # lock-release also fails, the original DELETE exception is
+ # the more useful one to propagate (Redis is clearly in
+ # trouble and the original error captures that).
+ with contextlib.suppress(Exception):
+ redis_client.delete(_abort_lock_key(execution_id))
+ raise
+ logger.error(
+ f"[exec:{execution_id}] Header task failed; barrier aborted. "
+ f"Cleaned up remaining/results keys. exc={exc!r}"
+ )
+ return {
+ "status": "aborted",
+ "execution_id": execution_id,
+ "reason": str(exc),
+ }
diff --git a/workers/sample.env b/workers/sample.env
index 5c0bce2d4..4764fb0c5 100644
--- a/workers/sample.env
+++ b/workers/sample.env
@@ -92,6 +92,45 @@ CACHE_REDIS_SSL_CERT_REQS=required
# When True: point CACHE_REDIS_HOST to Sentinel K8s service DNS, CACHE_REDIS_PORT to 26379
CACHE_REDIS_SENTINEL_MODE=False
+# =============================================================================
+# Distributed Barrier
+# =============================================================================
+# Selects the substrate that coordinates chord-style fan-out + callback:
+# chord (default) — celery.chord(header)(body); current production behaviour
+# redis — Redis DECR remaining:{exec_id} + RPUSH results
+# aggregation. Replaces only the chord aggregation
+# primitive; dispatch / retries / monitoring still ride
+# on Celery. Avoids the documented silent-drop class
+# in Celery's chord backend at large scale.
+# Default chord keeps existing behaviour exactly — unset / missing env
+# is safe.
+WORKER_BARRIER_BACKEND=chord
+
+# Optional separate Redis connection for the barrier counter +
+# results-aggregation keys. Falls back to REDIS_* (the canonical worker
+# Redis instance) if unset. Set these only if you want the barrier on a
+# different Redis from the cache layer.
+# WORKER_BARRIER_REDIS_HOST=unstract-redis
+# WORKER_BARRIER_REDIS_PORT=6379
+# WORKER_BARRIER_REDIS_DB=0
+# WORKER_BARRIER_REDIS_PASSWORD=
+# WORKER_BARRIER_REDIS_SENTINEL_MODE=False
+
+# Belt-and-suspenders cleanup TTL for orphaned barrier counter +
+# results keys. On the happy path the keys are explicitly DEL'd when
+# the last task completes — TTL only matters when the counter stalls
+# (e.g. some header tasks fail and the link never fires for them).
+#
+# MUST be strictly > longest plausible execution wall-clock. If TTL
+# expires while tasks are still in flight, the next successful task
+# fires a spurious callback with just its own result.
+#
+# 6h default gives 2-3x margin over FILE_PROCESSING_TASK_TIME_LIMIT
+# (this sample ships 2h; docker/sample.env up to 3h; process_file_batch
+# has max_retries=0 so no retry amplification). Raise this if your
+# workflows can plausibly span >6h.
+WORKER_BARRIER_KEY_TTL_SECONDS=21600
+
# Database URL (for fallback usage)
DATABASE_URL=postgresql://unstract_dev:unstract_pass@unstract-db:5432/unstract_db
diff --git a/workers/shared/workflow/execution/orchestration_utils.py b/workers/shared/workflow/execution/orchestration_utils.py
index 6d4cf7ff4..5b7167774 100644
--- a/workers/shared/workflow/execution/orchestration_utils.py
+++ b/workers/shared/workflow/execution/orchestration_utils.py
@@ -4,22 +4,29 @@ This module provides standardized workflow orchestration patterns,
chord execution, batch processing, and task coordination utilities.
"""
+from __future__ import annotations
+
import os
-from typing import Any
+from typing import TYPE_CHECKING, Any
-from queue_backend import CeleryChordBarrier, FairnessKey
+from queue_backend import BarrierHandle, FairnessKey, get_barrier
from ...enums import FileDestinationType, PipelineType
from ...enums.worker_enums import QueueName
from ...infrastructure.logging import WorkerLogger
+if TYPE_CHECKING:
+ from celery.canvas import Signature
+
logger = WorkerLogger.get_logger(__name__)
# Single ``Barrier`` instance reused across all
-# ``WorkflowOrchestrationUtils.create_chord_execution`` calls. A future
-# Phase 6b will replace this with a factory call (``get_barrier()``)
-# that picks the impl from ``WORKER_BARRIER_BACKEND``.
-_BARRIER = CeleryChordBarrier()
+# ``WorkflowOrchestrationUtils.create_chord_execution`` calls. The
+# substrate is selected at module-import (worker startup) time by the
+# ``WORKER_BARRIER_BACKEND`` env var (default ``chord``). Flag flips
+# require a pod restart — same posture as every other ``WORKER_*``
+# env in the codebase.
+_BARRIER = get_barrier()
class WorkflowOrchestrationUtils:
@@ -27,14 +34,14 @@ class WorkflowOrchestrationUtils:
@staticmethod
def create_chord_execution(
- batch_tasks: list[Any],
+ batch_tasks: list[Signature],
callback_task_name: str,
callback_kwargs: dict[str, Any],
callback_queue: str,
app_instance: Any,
*,
fairness: FairnessKey | None = None,
- ) -> Any:
+ ) -> BarrierHandle | None:
"""Standardized fan-out + callback pattern (Phase 6 ``Barrier``).
Routes through ``CeleryChordBarrier`` — a thin wrapper around
diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.py
index 8d24658ca..e61bc4a18 100644
--- a/workers/tests/test_barrier.py
+++ b/workers/tests/test_barrier.py
@@ -28,7 +28,6 @@ The single ``chord(...)`` call still lives in
from __future__ import annotations
-import contextlib
from dataclasses import dataclass
from unittest.mock import MagicMock, patch
@@ -105,12 +104,13 @@ class TestBarrierProtocolShape:
#
# ``required_attrs`` is hand-maintained: if a future refactor
# adds a required attribute to ``TaskHandle`` /
- # ``BarrierHandle``, add it here too. We do *not* introspect
- # ``TaskHandle.__annotations__`` because runtime-checkable
- # Protocols would force every substrate handle (including
- # third-party ones we don't control) to be runtime-checkable
- # too. Trade-off: the maintenance is manual but the contract
- # is enforced.
+ # ``BarrierHandle``, add it here too. The hand-maintained form
+ # is grepable next to the comment that explains it, and
+ # ``TaskHandle`` is intentionally minimal (``id: str``) so the
+ # manual-update burden is one line every several phases.
+ # ``__annotations__`` introspection would also work (and
+ # doesn't require ``@runtime_checkable``), but the explicit
+ # tuple keeps the contract surface explicit.
async_result_instance = AsyncResult("placeholder-task-id")
required_attrs = ("id",)
missing = [
@@ -557,10 +557,8 @@ def _setup_workflow_api_mocks(
mock_create_chord = MagicMock(name="create_chord_execution")
# Truthy handle on the success path; ``None`` on the two failure
- # paths. Production's ``if not result:`` branch (today; a future
- # phase-6b PR may tighten to ``if result is None:`` — see the
- # TODO at the call site in ``api-deployment/tasks.py``) then
- # distinguishes the two ``None`` cases via ``batch_tasks`` length.
+ # paths. Production's ``if result is None:`` branch then
+ # distinguishes the two cases via ``batch_tasks`` length.
if chord_outcome == "success":
mock_create_chord.return_value = MagicMock(id="chord-result-id")
else:
@@ -720,12 +718,13 @@ class TestCallSiteFairnessContracts:
mock_create_chord,
)
- # ``contextlib.suppress`` over a bare ``except Exception: pass``
- # to declare intent explicitly: any post-``create_chord_execution``
- # raise (e.g. the mocked api_client's status update path) is
- # tolerated, because the next ``assert mock_create_chord.called``
- # is what proves the chord call was actually reached.
- with contextlib.suppress(Exception):
+ # Tolerate post-``create_chord_execution`` raises (e.g. the
+ # mocked api_client's status-update path), but re-raise if
+ # ``create_chord_execution`` was never invoked — a pre-chord
+ # regression would otherwise be silently swallowed and surface
+ # as a misleading ``assert mock_create_chord.called`` failure
+ # rather than the actual error.
+ try:
general_tasks._orchestrate_file_processing_general(
api_client=api_client,
workflow_id="wf-1",
@@ -737,6 +736,9 @@ class TestCallSiteFairnessContracts:
use_file_history=False,
organization_id="org_test",
)
+ except Exception:
+ if not mock_create_chord.called:
+ raise
assert mock_create_chord.called, (
"_orchestrate_file_processing_general did not call "
diff --git a/workers/tests/test_barrier_backend_selection.py b/workers/tests/test_barrier_backend_selection.py
new file mode 100644
index 000000000..7a452652e
--- /dev/null
+++ b/workers/tests/test_barrier_backend_selection.py
@@ -0,0 +1,116 @@
+"""Tests for the ``get_barrier()`` factory + ``WORKER_BARRIER_BACKEND`` flag.
+
+Phase 6b selects the ``Barrier`` substrate at runtime via an env flag.
+This test suite pins:
+
+- Default (env unset) → ``CeleryChordBarrier`` — protects against any
+ production behaviour change at merge time; downstream envs that
+ haven't added the new flag still run the same substrate they ran
+ yesterday.
+- ``WORKER_BARRIER_BACKEND=chord`` → ``CeleryChordBarrier``.
+- ``WORKER_BARRIER_BACKEND=redis`` → ``RedisDecrBarrier``.
+- Unknown values raise ``ValueError`` — silent fallback to default
+ would mask a typo'd flag in production env, which the canary
+ must not allow.
+- Case + whitespace tolerance — operators frequently set env vars
+ via shell scripts or k8s configmaps; ``" REDIS "`` should
+ resolve the same as ``"redis"``.
+- ``BarrierBackend`` enum membership — the canonical string values
+ match what operators set in env.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from queue_backend import (
+ BarrierBackend,
+ CeleryChordBarrier,
+ RedisDecrBarrier,
+ get_barrier,
+)
+
+
+class TestBarrierBackendEnum:
+ def test_chord_value_is_canonical_string(self):
+ """The enum value is what operators set in env. If this
+ drifts, the factory's dispatch breaks silently for one
+ substrate (the one whose enum value no longer matches the
+ documented env value)."""
+ assert BarrierBackend.CHORD.value == "chord"
+ assert BarrierBackend.REDIS.value == "redis"
+
+ def test_str_enum_string_equality(self):
+ """``StrEnum`` members compare equal to their string values —
+ means tests + production code can use either form
+ interchangeably without conversion."""
+ assert BarrierBackend.CHORD == "chord"
+ assert BarrierBackend.REDIS == "redis"
+
+
+class TestGetBarrierFactory:
+ def test_default_returns_celery_chord_barrier(self, monkeypatch):
+ """Env unset → ``CeleryChordBarrier``. This is what protects
+ production from a behaviour change at merge time — downstream
+ envs (``unstract-cloud``, self-hosted, etc.) that haven't
+ added the new flag run the same substrate they ran yesterday."""
+ monkeypatch.delenv("WORKER_BARRIER_BACKEND", raising=False)
+ barrier = get_barrier()
+ assert isinstance(barrier, CeleryChordBarrier)
+
+ def test_chord_selects_celery_chord_barrier(self, monkeypatch):
+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.CHORD)
+ assert isinstance(get_barrier(), CeleryChordBarrier)
+
+ def test_redis_selects_redis_decr_barrier(self, monkeypatch):
+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.REDIS)
+ assert isinstance(get_barrier(), RedisDecrBarrier)
+
+ def test_unknown_value_raises_loudly(self, monkeypatch):
+ """Silent fallback to default would mask a typo'd production
+ env (``rediz``, ``REDDIS``, ``redis-decr``) — the canary
+ must surface the misconfiguration."""
+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", "rediz")
+ with pytest.raises(ValueError, match=r"rediz"):
+ get_barrier()
+
+ def test_empty_string_raises_loudly(self, monkeypatch):
+ """Empty string is not the same as "unset" — it's a positive
+ misconfiguration (e.g. an unsubstituted ``$BARRIER`` in a
+ shell script). Surface it rather than falling back to
+ default, which would mask the bug."""
+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", "")
+ with pytest.raises(ValueError):
+ get_barrier()
+
+ def test_case_insensitive(self, monkeypatch):
+ """Operators frequently set env vars via shell heredocs or
+ k8s configmaps with inconsistent casing. ``REDIS`` and
+ ``Redis`` should resolve the same as ``redis``."""
+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", "REDIS")
+ assert isinstance(get_barrier(), RedisDecrBarrier)
+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", "Chord")
+ assert isinstance(get_barrier(), CeleryChordBarrier)
+
+ def test_whitespace_tolerated(self, monkeypatch):
+ """Stray whitespace (common from shell substitutions) is
+ stripped before the membership check."""
+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", " redis ")
+ assert isinstance(get_barrier(), RedisDecrBarrier)
+
+ def test_returns_fresh_instance_per_call(self, monkeypatch):
+ """``get_barrier()`` builds a new instance each call rather
+ than returning a singleton. Call sites that want a singleton
+ (e.g. ``orchestration_utils._BARRIER``) cache it at module
+ load time. Tests can flip the env per-test without leaking
+ a stale instance from a previous test."""
+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", "chord")
+ a = get_barrier()
+ b = get_barrier()
+ assert a is not b
+ assert isinstance(a, CeleryChordBarrier)
+ assert isinstance(b, CeleryChordBarrier)
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/workers/tests/test_barrier_differential.py b/workers/tests/test_barrier_differential.py
new file mode 100644
index 000000000..99bbb1200
--- /dev/null
+++ b/workers/tests/test_barrier_differential.py
@@ -0,0 +1,232 @@
+"""Differential test — both ``Barrier`` substrates produce equivalent
+callback firings for a fixed input.
+
+This is the load-bearing test that justifies the substrate swap:
+given the same header tasks + callback descriptor + fairness, both
+``CeleryChordBarrier`` and ``RedisDecrBarrier`` must ultimately fire
+the same callback with the same ``(task_name, args, kwargs, queue,
+headers)``. The execution paths diverge — chord aggregates via
+Celery's chord backend, redis aggregates via ``DECR remaining`` +
+``RPUSH`` — but the observable output (what the callback receives)
+is equivalent.
+
+If this test fails, the substrate swap is *not* safe to roll out:
+the callback would see different aggregated results or different
+metadata between substrates, breaking executions when the flag
+flips.
+
+Note this is a unit-level differential — it pins the wire shape of
+the callback dispatch, not full end-to-end equivalence. A future
+integration test (against real Celery + Redis with N parallel
+workers) would belong in ``test_barrier_integration.py``.
+"""
+
+from __future__ import annotations
+
+import json
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from queue_backend import CeleryChordBarrier, RedisDecrBarrier, barrier_decr_and_check
+from queue_backend.fairness import FairnessKey, WorkloadType
+
+
+# A fixed scenario both substrates run through.
+_HEADER_TASK_RESULTS = [
+ {"batch_id": "b1", "files": ["f1", "f2"], "status": "ok"},
+ {"batch_id": "b2", "files": ["f3", "f4"], "status": "ok"},
+ {"batch_id": "b3", "files": ["f5"], "status": "ok"},
+]
+_CALLBACK_TASK = "process_batch_callback_api"
+_CALLBACK_KWARGS = {
+ "execution_id": "exec-differential-1",
+ "pipeline_id": "pipe-x",
+ "organization_id": "org-y",
+}
+_CALLBACK_QUEUE = "api_file_processing_callback"
+_FAIRNESS = FairnessKey(org_id="org-y", workload_type=WorkloadType.API)
+
+
+def _capture_chord_callback_dispatch() -> dict:
+ """Run ``CeleryChordBarrier.enqueue`` against a mocked ``chord``
+ and return the callback signature's construction args.
+
+ Chord's wire model: ``chord(header_tasks)(callback_signature)`` —
+ the callback_signature is built via ``app.signature(name, kwargs=...,
+ queue=..., headers=...)`` and then passed as the body.
+ """
+ app = MagicMock(name="app")
+ headers_captured = {}
+
+ def signature_recorder(name, **kwargs):
+ headers_captured["task_name"] = name
+ headers_captured["args"] = kwargs.get("args") # None for chord callback
+ headers_captured["kwargs"] = kwargs.get("kwargs")
+ headers_captured["queue"] = kwargs.get("queue")
+ headers_captured["headers"] = kwargs.get("headers")
+ return MagicMock(name="callback_signature")
+
+ app.signature.side_effect = signature_recorder
+ header = [MagicMock(name=f"h{i}") for i in range(len(_HEADER_TASK_RESULTS))]
+
+ with patch("queue_backend.barrier.chord"):
+ CeleryChordBarrier().enqueue(
+ header,
+ callback_task_name=_CALLBACK_TASK,
+ callback_kwargs=_CALLBACK_KWARGS,
+ callback_queue=_CALLBACK_QUEUE,
+ app_instance=app,
+ fairness=_FAIRNESS,
+ )
+ return headers_captured
+
+
+def _capture_redis_callback_dispatch() -> dict:
+ """Drive ``RedisDecrBarrier.enqueue`` + simulate the link task
+ completing for all header tasks, then capture the final
+ callback signature's construction args at the post-aggregation
+ dispatch site.
+ """
+ redis_mock = MagicMock(name="redis")
+ barrier = RedisDecrBarrier(redis_client=redis_mock)
+ header = [MagicMock(name=f"h{i}") for i in range(len(_HEADER_TASK_RESULTS))]
+ barrier.enqueue(
+ header,
+ callback_task_name=_CALLBACK_TASK,
+ callback_kwargs=_CALLBACK_KWARGS,
+ callback_queue=_CALLBACK_QUEUE,
+ app_instance=MagicMock(name="app"),
+ fairness=_FAIRNESS,
+ )
+
+ # Now simulate each link firing. The first N-1 see remaining > 0;
+ # the last sees remaining == 0 and dispatches the callback.
+ headers_captured = {}
+
+ def signature_recorder(name, **kwargs):
+ headers_captured["task_name"] = name
+ headers_captured["args"] = kwargs.get("args")
+ headers_captured["kwargs"] = kwargs.get("kwargs")
+ headers_captured["queue"] = kwargs.get("queue")
+ headers_captured["headers"] = kwargs.get("headers")
+ cb_sig = MagicMock(name="callback_signature")
+ cb_sig.apply_async.return_value = MagicMock(id="callback-task-id")
+ return cb_sig
+
+ # Simulate the Lua script's return for each link invocation.
+ # First N-1 calls: [remaining, []]. Last call: [0, [serialised results]].
+ n = len(_HEADER_TASK_RESULTS)
+ serialised_results = [json.dumps(r) for r in _HEADER_TASK_RESULTS]
+ return_values = [
+ [n - 1 - i, []] for i in range(n - 1)
+ ] + [[0, serialised_results]]
+ script_mock = MagicMock(side_effect=return_values)
+
+ with patch("queue_backend.redis_barrier._get_redis_client") as get_redis:
+ # Inside the link task, ``_get_redis_client()`` is called to
+ # build the script invocation. Return a mock whose
+ # register_script returns the script_mock that yields the
+ # sequenced return values.
+ link_redis = MagicMock(name="link_redis")
+ link_redis.register_script.return_value = script_mock
+ get_redis.return_value = link_redis
+
+ with patch("celery.current_app") as current_app:
+ current_app.signature.side_effect = signature_recorder
+ for header_result in _HEADER_TASK_RESULTS:
+ barrier_decr_and_check.run(
+ header_result,
+ execution_id=_CALLBACK_KWARGS["execution_id"],
+ callback_descriptor={
+ "task_name": _CALLBACK_TASK,
+ "kwargs": _CALLBACK_KWARGS,
+ "queue": _CALLBACK_QUEUE,
+ "fairness_headers": _FAIRNESS.as_header(),
+ },
+ )
+ return headers_captured
+
+
+class TestBarrierSubstrateDifferential:
+ """Both substrates must produce equivalent callback firings.
+
+ A failure here means the swap is not safe — flag flipping in
+ Phase 6c would change the data the callback receives.
+ """
+
+ def test_same_callback_task_name(self):
+ """Both substrates fire the same callback task name."""
+ chord_cap = _capture_chord_callback_dispatch()
+ redis_cap = _capture_redis_callback_dispatch()
+ assert chord_cap["task_name"] == redis_cap["task_name"] == _CALLBACK_TASK
+
+ def test_same_callback_kwargs(self):
+ """Both substrates forward the same ``execution_id`` /
+ ``pipeline_id`` / ``organization_id`` to the callback."""
+ chord_cap = _capture_chord_callback_dispatch()
+ redis_cap = _capture_redis_callback_dispatch()
+ assert chord_cap["kwargs"] == redis_cap["kwargs"] == _CALLBACK_KWARGS
+
+ def test_same_callback_queue(self):
+ """Both substrates route the callback to the same queue —
+ without this, post-Phase-6c the callback could land on a
+ worker that doesn't have it registered."""
+ chord_cap = _capture_chord_callback_dispatch()
+ redis_cap = _capture_redis_callback_dispatch()
+ assert chord_cap["queue"] == redis_cap["queue"] == _CALLBACK_QUEUE
+
+ def test_same_fairness_headers(self):
+ """Both substrates ride the same ``x-fairness-key`` AMQP
+ header on the callback. Mismatch would mean post-flag-flip
+ executions route through different fairness slots, which
+ the L1/L2/L3 admission logic could throttle differently."""
+ chord_cap = _capture_chord_callback_dispatch()
+ redis_cap = _capture_redis_callback_dispatch()
+ assert chord_cap["headers"] == redis_cap["headers"] == _FAIRNESS.as_header()
+
+ def test_redis_substrate_passes_aggregated_results_as_first_arg(self):
+ """Chord's callback receives ``[result_1, result_2, ...]`` via
+ Celery's chord-aggregator (an implicit first positional arg
+ Celery injects). The Redis substrate must replicate the
+ same shape — pass the aggregated list explicitly via
+ ``args=[all_results]`` on the callback signature.
+
+ We can't observe chord's implicit arg injection from the
+ outside (it's the chord backend's job), but we CAN pin the
+ Redis substrate's behaviour: when aggregation completes,
+ the callback signature's ``args[0]`` MUST equal the
+ original header-task results list. That's the contract the
+ callback already depends on (``file_batch_results: list[dict]``
+ — see callback/tasks.py:1552)."""
+ redis_cap = _capture_redis_callback_dispatch()
+ assert redis_cap["args"] == [_HEADER_TASK_RESULTS], (
+ f"Redis substrate's callback args must be a list "
+ f"containing one list-of-results (matching chord's "
+ f"aggregator shape). Got: {redis_cap['args']!r}"
+ )
+
+ def test_both_handle_empty_headers_identically(self):
+ """Empty header → ``None`` from both substrates. The caller's
+ ``if result is None:`` branch then handles the zero-files
+ contract uniformly."""
+ chord_result = CeleryChordBarrier().enqueue(
+ [],
+ callback_task_name=_CALLBACK_TASK,
+ callback_kwargs=_CALLBACK_KWARGS,
+ callback_queue=_CALLBACK_QUEUE,
+ app_instance=MagicMock(),
+ )
+ redis_result = RedisDecrBarrier(redis_client=MagicMock()).enqueue(
+ [],
+ callback_task_name=_CALLBACK_TASK,
+ callback_kwargs=_CALLBACK_KWARGS,
+ callback_queue=_CALLBACK_QUEUE,
+ app_instance=MagicMock(),
+ )
+ assert chord_result is None
+ assert redis_result is None
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/workers/tests/test_queue_backend_seam.py b/workers/tests/test_queue_backend_seam.py
index 7075dbaf2..8cd827cc1 100644
--- a/workers/tests/test_queue_backend_seam.py
+++ b/workers/tests/test_queue_backend_seam.py
@@ -276,14 +276,21 @@ class TestPublicSurface:
import queue_backend
# Phase 6a added Barrier / BarrierHandle / CeleryChordBarrier.
- # Phase 6b will add a ``get_barrier`` factory when the
- # ``WORKER_BARRIER_BACKEND`` flag wiring lands.
+ # Phase 6b adds RedisDecrBarrier + barrier_decr_and_check
+ # (registered as a Celery task on import) + the BarrierBackend
+ # enum + the get_barrier factory that the WORKER_BARRIER_BACKEND
+ # env flag drives.
assert set(queue_backend.__all__) == {
"Barrier",
+ "BarrierBackend",
"BarrierHandle",
"CeleryChordBarrier",
"FairnessKey",
+ "RedisDecrBarrier",
+ "barrier_abort",
+ "barrier_decr_and_check",
"dispatch",
+ "get_barrier",
"worker_task",
}
diff --git a/workers/tests/test_redis_barrier.py b/workers/tests/test_redis_barrier.py
new file mode 100644
index 000000000..de098cf58
--- /dev/null
+++ b/workers/tests/test_redis_barrier.py
@@ -0,0 +1,972 @@
+"""Characterisation tests for ``RedisDecrBarrier`` (PG Queue Phase 6b).
+
+The risky behaviour-change PR replacing Celery's chord aggregation
+primitive with the labs-design ``DECR remaining`` + ``RPUSH results``
+pattern (see ``queue_backend/redis_barrier.py``).
+
+Three layers:
+
+1. **Protocol shape** — ``RedisDecrBarrier`` satisfies the ``Barrier``
+ Protocol; the handle satisfies ``BarrierHandle``.
+2. **Wire model** — ``enqueue`` initialises the per-execution counter
+ + results list with TTL, stamps fairness on each header task,
+ attaches the ``barrier_decr_and_check`` link, and dispatches via
+ ``apply_async``.
+3. **Link aggregation** — the ``barrier_decr_and_check`` task runs the
+ atomic Lua script, fires the callback exactly when remaining reads
+ 0, passes the aggregated results, and respects fairness on the
+ callback signature.
+
+Redis is mocked via ``MagicMock`` — the contract under test is the
+sequence of Redis ops + their args, not Redis's own correctness.
+A future integration test (against a real Redis) would belong in
+``test_redis_barrier_integration.py``.
+"""
+
+from __future__ import annotations
+
+import json
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from queue_backend import (
+ Barrier,
+ BarrierHandle,
+ RedisDecrBarrier,
+ barrier_decr_and_check,
+)
+from queue_backend.fairness import FAIRNESS_HEADER_NAME, FairnessKey, WorkloadType
+from queue_backend.redis_barrier import (
+ _KEY_TTL_DEFAULT_SECONDS,
+ _remaining_key,
+ _results_key,
+)
+
+
+# --- Protocol shape ---
+
+
+class TestRedisDecrBarrierProtocolShape:
+ def test_satisfies_barrier_protocol(self):
+ barrier: Barrier = RedisDecrBarrier()
+ assert callable(getattr(barrier, "enqueue", None))
+
+ def test_handle_satisfies_barrier_handle(self):
+ from queue_backend.redis_barrier import _RedisBarrierHandle
+
+ handle: BarrierHandle = _RedisBarrierHandle(id="exec-1")
+ assert handle.id == "exec-1"
+ assert isinstance(handle.id, str)
+
+
+# --- Enqueue: wire model ---
+
+
+def redis_mock():
+ """Mock the Redis client used by ``RedisDecrBarrier``."""
+ return MagicMock(name="redis_client")
+
+
+def barrier(redis_mock):
+ """``RedisDecrBarrier`` with an injected mock client.
+
+ Production builds its client from env via ``create_redis_client``;
+ tests inject so no Redis is required.
+ """
+ return RedisDecrBarrier(redis_client=redis_mock)
+
+
+class TestRedisDecrBarrierEnqueue:
+ def test_empty_header_returns_none_and_touches_no_redis(self, barrier, redis_mock):
+ """Mirrors the ``CeleryChordBarrier`` zero-header contract:
+ ``None`` is the *sole* signal of no-work-enqueued, and we
+ don't pay the Redis round-trip for it."""
+ result = barrier.enqueue(
+ [],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+ assert result is None
+ # No SET / DELETE on the empty path — short-circuit before any
+ # Redis round-trip.
+ redis_mock.set.assert_not_called()
+ redis_mock.delete.assert_not_called()
+
+ def test_missing_execution_id_raises(self, barrier):
+ """``RedisDecrBarrier`` uses ``execution_id`` as the key suffix
+ for ``remaining:{exec_id}`` / ``results:{exec_id}``. Without
+ it we can't isolate this execution's counter — fail loudly
+ instead of routing into a global namespace collision."""
+ header = [MagicMock(name="h1")]
+ with pytest.raises(ValueError, match=r"execution_id"):
+ barrier.enqueue(
+ header,
+ callback_task_name="cb",
+ # No execution_id.
+ callback_kwargs={"pipeline_id": "p-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+
+ def test_initialises_remaining_counter_with_ttl(self, barrier, redis_mock):
+ """``remaining:{exec_id}`` SET to ``len(header_tasks)`` with
+ TTL — belt-and-suspenders cleanup if a link never fires for
+ a task (e.g. worker crash mid-execution)."""
+ header = [MagicMock(name="h1"), MagicMock(name="h2"), MagicMock(name="h3")]
+ barrier.enqueue(
+ header,
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-42"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+ redis_mock.set.assert_called_once_with(
+ _remaining_key("exec-42"),
+ 3,
+ ex=_KEY_TTL_DEFAULT_SECONDS,
+ )
+ # ``results:{exec_id}`` AND ``abort_lock:{exec_id}`` are DEL'd
+ # to clear any stale prior-run leftovers (e.g. from a retry
+ # reusing the same execution_id). Without DELing the abort
+ # lock here, a stale lock from a prior failed run would mask
+ # a retry's failure as success.
+ from queue_backend.redis_barrier import _abort_lock_key
+
+ redis_mock.delete.assert_called_once_with(
+ _results_key("exec-42"),
+ _abort_lock_key("exec-42"),
+ )
+
+ def test_link_attached_to_each_header_task(self, barrier, redis_mock):
+ """Every header task gets the ``barrier_decr_and_check`` link.
+ Cloning (not mutating the caller's list) preserves the same
+ guarantee ``CeleryChordBarrier`` makes for cross-tenant
+ signature reuse."""
+ h1 = MagicMock(name="h1")
+ h2 = MagicMock(name="h2")
+ barrier.enqueue(
+ [h1, h2],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+ # Each task cloned, .link(...) called once on the clone,
+ # apply_async called once on the clone.
+ h1.clone.assert_called_once_with()
+ h2.clone.assert_called_once_with()
+ h1.clone.return_value.link.assert_called_once()
+ h2.clone.return_value.link.assert_called_once()
+ h1.clone.return_value.apply_async.assert_called_once_with()
+ h2.clone.return_value.apply_async.assert_called_once_with()
+ # The caller's signatures were NOT mutated (no .link on the
+ # originals).
+ h1.link.assert_not_called()
+ h2.link.assert_not_called()
+
+ def test_fairness_header_stamped_on_each_header_task(self, barrier, redis_mock):
+ """``fairness.as_header()`` rides on every header signature as
+ an additive AMQP header — same wire shape as
+ ``CeleryChordBarrier``'s fairness plumbing."""
+ h1 = MagicMock(name="h1")
+ fairness = FairnessKey(org_id="org-x", workload_type=WorkloadType.API)
+ barrier.enqueue(
+ [h1],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ fairness=fairness,
+ )
+ # ``as_header()`` returns ``{FAIRNESS_HEADER_NAME: <dict>}``
+ # (a nested dict, not a JSON string — see fairness.py:62).
+ h1.clone.return_value.set.assert_called_once_with(
+ headers=fairness.as_header()
+ )
+
+ def test_no_fairness_no_header_added(self, barrier, redis_mock):
+ """When ``fairness=None``, no ``headers=`` kwarg on the
+ signature — preserves wire equivalence for the
+ non-fairness call path."""
+ h1 = MagicMock(name="h1")
+ barrier.enqueue(
+ [h1],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+ h1.clone.return_value.set.assert_not_called()
+
+ def test_returns_handle_with_execution_id(self, barrier, redis_mock):
+ """``BarrierHandle.id`` is the execution id — what call sites
+ log for chord-id tracing. (Under ``CeleryChordBarrier`` this
+ was the chord aggregator's AsyncResult id; under
+ ``RedisDecrBarrier`` we don't have such a task, so we expose
+ the execution id which is a stable per-execution identifier
+ that the existing log consumers don't depend on the
+ underlying type of.)"""
+ result = barrier.enqueue(
+ [MagicMock()],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-42"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+ assert result is not None
+ assert result.id == "exec-42"
+ assert isinstance(result.id, str)
+
+ def test_setup_failure_raises(self, barrier, redis_mock):
+ """Any substrate failure (Redis down, network blip mid-setup)
+ propagates as an exception — matches the ``Barrier`` Protocol:
+ ``None`` is sole "no-op" signal, everything else raises."""
+ redis_mock.set.side_effect = ConnectionError("redis is down")
+ with pytest.raises(ConnectionError, match="redis is down"):
+ barrier.enqueue(
+ [MagicMock()],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+
+ def test_default_ttl_is_six_hours(self):
+ """Lock in the chosen default. Accidental shrinkage to a few
+ minutes would cause spurious callback fires on any execution
+ longer than the new value; accidental expansion to days would
+ leak orphaned keys. The default reflects the worst-case
+ ``FILE_PROCESSING_TASK_TIME_LIMIT`` (3h) x 2x margin."""
+ assert _KEY_TTL_DEFAULT_SECONDS == 6 * 60 * 60
+
+ def test_ttl_overridable_via_env(self, barrier, redis_mock, monkeypatch):
+ """Operators with substantially longer (or shorter) workflows
+ can override the cleanup TTL via ``WORKER_BARRIER_KEY_TTL_SECONDS``.
+
+ Set well above the default to confirm the env-driven path is
+ actually read (not the constant)."""
+ monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", str(3 * 24 * 3600))
+ barrier.enqueue(
+ [MagicMock()],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+ # ``ex=`` on the counter SET should be the overridden 3-day value.
+ assert redis_mock.set.call_args.kwargs.get("ex") == 3 * 24 * 3600
+
+ # Invalid-value handling tests live in ``TestTtlEnvValidation``
+ # below — they now assert ``raises`` (matching ``get_barrier()``'s
+ # loud-on-misconfig posture) rather than silent fallback.
+
+
+# --- Link task: aggregation ---
+
+
+class TestBarrierDecrAndCheckLink:
+ """The ``barrier_decr_and_check`` worker task — the load-bearing
+ aggregator. Runs after each header task and either decrements +
+ aggregates (most calls) or fires the callback (the last call).
+ """
+
+ @patch("queue_backend.redis_barrier._get_redis_client")
+ def test_pending_path_decrement_only(self, get_redis):
+ """When the post-decrement counter reads >0, no callback
+ dispatch — just ``RPUSH + DECR`` and return a pending status."""
+ redis_client = MagicMock(name="redis")
+ # Lua script returns [remaining, results-empty-list-since-pending].
+ # ``register_script`` returns a callable that returns the script result.
+ script = MagicMock(name="script", return_value=[2, []])
+ redis_client.register_script.return_value = script
+ get_redis.return_value = redis_client
+
+ with patch("celery.current_app") as current_app:
+ # Invoke the underlying function directly (bypass
+ # Celery's @worker_task wrapper for unit testing).
+ result = barrier_decr_and_check.run(
+ {"batch_id": "b1", "status": "ok"},
+ execution_id="exec-1",
+ callback_descriptor={
+ "task_name": "cb",
+ "kwargs": {"execution_id": "exec-1"},
+ "queue": "q",
+ "fairness_headers": None,
+ },
+ )
+
+ # Lua script invoked with the right keys + JSON-serialised
+ # result + TTL seconds for the EXPIRE inside the script.
+ script.assert_called_once()
+ call_kwargs = script.call_args.kwargs
+ assert call_kwargs["keys"] == [
+ _remaining_key("exec-1"),
+ _results_key("exec-1"),
+ ]
+ assert json.loads(call_kwargs["args"][0]) == {"batch_id": "b1", "status": "ok"}
+ assert call_kwargs["args"][1] == _KEY_TTL_DEFAULT_SECONDS
+
+ # No callback dispatch — remaining > 0.
+ current_app.signature.assert_not_called()
+ assert result == {"status": "pending", "remaining": 2}
+
+ @patch("queue_backend.redis_barrier._get_redis_client")
+ def test_complete_path_fires_callback_with_aggregated_results(self, get_redis):
+ """When the post-decrement counter reads 0, the link reads the
+ full results list (returned by the Lua script in the same
+ atomic step), constructs the callback signature with the
+ aggregated list as the first arg, and dispatches via
+ ``apply_async``."""
+ redis_client = MagicMock(name="redis")
+ # Lua returns [0, [serialised result_1, result_2, result_3]].
+ aggregated_raw = [
+ json.dumps({"batch_id": "b1", "status": "ok"}),
+ json.dumps({"batch_id": "b2", "status": "ok"}),
+ json.dumps({"batch_id": "b3", "status": "ok"}),
+ ]
+ script = MagicMock(name="script", return_value=[0, aggregated_raw])
+ redis_client.register_script.return_value = script
+ get_redis.return_value = redis_client
+
+ with patch("celery.current_app") as current_app:
+ cb_sig = MagicMock(name="callback_signature")
+ cb_sig.apply_async.return_value = MagicMock(id="callback-task-id-xyz")
+ current_app.signature.return_value = cb_sig
+
+ result = barrier_decr_and_check.run(
+ {"batch_id": "b3", "status": "ok"},
+ execution_id="exec-42",
+ callback_descriptor={
+ "task_name": "process_batch_callback_api",
+ "kwargs": {
+ "execution_id": "exec-42",
+ "pipeline_id": "pipe-7",
+ "organization_id": "org-x",
+ },
+ "queue": "api_file_processing_callback",
+ "fairness_headers": None,
+ },
+ )
+
+ # Callback signature constructed with aggregated results as
+ # first arg, callback kwargs forwarded, queue pinned.
+ current_app.signature.assert_called_once_with(
+ "process_batch_callback_api",
+ args=[
+ [
+ {"batch_id": "b1", "status": "ok"},
+ {"batch_id": "b2", "status": "ok"},
+ {"batch_id": "b3", "status": "ok"},
+ ],
+ ],
+ kwargs={
+ "execution_id": "exec-42",
+ "pipeline_id": "pipe-7",
+ "organization_id": "org-x",
+ },
+ queue="api_file_processing_callback",
+ )
+ cb_sig.apply_async.assert_called_once_with()
+ # DELETE is now called from Python AFTER apply_async succeeds
+ # (the Lua script no longer DELs in the ``== 0`` branch). This
+ # is the load-bearing deferred-cleanup invariant: a dispatch
+ # failure leaves the keys + TTL in place so the execution
+ # isn't stranded.
+ redis_client.delete.assert_called_once_with(
+ _remaining_key("exec-42"),
+ _results_key("exec-42"),
+ )
+
+ assert result["status"] == "complete"
+ assert result["callback_task_id"] == "callback-task-id-xyz"
+ assert result["aggregated_count"] == 3
+
+ @patch("queue_backend.redis_barrier._get_redis_client")
+ def test_complete_path_passes_fairness_header_to_callback(self, get_redis):
+ """Fairness rides the callback signature too — same wire shape
+ as ``CeleryChordBarrier``'s callback fairness plumbing."""
+ redis_client = MagicMock(name="redis")
+ script = MagicMock(name="script", return_value=[0, [json.dumps({"x": 1})]])
+ redis_client.register_script.return_value = script
+ get_redis.return_value = redis_client
+
+ # Match the wire shape produced by ``FairnessKey.as_header()`` —
+ # a nested dict, not a JSON string (see fairness.py:62).
+ fairness_headers = {
+ FAIRNESS_HEADER_NAME: {
+ "org_id": "org-x",
+ "workload_type": "non_api",
+ "pipeline_priority": 5,
+ }
+ }
+ with patch("celery.current_app") as current_app:
+ current_app.signature.return_value = MagicMock(
+ apply_async=MagicMock(return_value=MagicMock(id="cb-id"))
+ )
+
+ barrier_decr_and_check.run(
+ {"x": 1},
+ execution_id="exec-1",
+ callback_descriptor={
+ "task_name": "cb",
+ "kwargs": {"execution_id": "exec-1"},
+ "queue": "q",
+ "fairness_headers": fairness_headers,
+ },
+ )
+
+ # ``headers=fairness_headers`` MUST be on the signature kwargs.
+ assert current_app.signature.call_args.kwargs.get("headers") == fairness_headers
+
+ @patch("queue_backend.redis_barrier._get_redis_client")
+ def test_unserialisable_result_raises_loudly(self, get_redis):
+ """A header task returning something json.dumps can't handle
+ is a typed-boundary regression. Raise loudly so the
+ execution surfaces an error rather than silently dropping the
+ result."""
+ redis_client = MagicMock(name="redis")
+ get_redis.return_value = redis_client
+
+ # ``complex`` is a classic JSON-unserialisable leaf — no
+ # ``__json__`` / ``__str__``-with-meaning, and ``default=str``
+ # isn't applied (we dropped it). A regressed
+ # ``BatchExecutionResult.to_dict()`` producing such a leaf
+ # would land here. The test pins that the loud-failure
+ # ``try/except`` actually surfaces the regression rather than
+ # being a no-op.
+ with pytest.raises((TypeError, ValueError)):
+ barrier_decr_and_check.run(
+ {"bad": complex(1, 2)},
+ execution_id="exec-1",
+ callback_descriptor={
+ "task_name": "cb",
+ "kwargs": {"execution_id": "exec-1"},
+ "queue": "q",
+ "fairness_headers": None,
+ },
+ )
+
+ def test_link_task_is_registered_with_celery_under_canonical_name(self):
+ """The link task must be importable + registered under
+ ``barrier_decr_and_check`` — any worker that processes the
+ link task's queue needs the name to resolve.
+
+ Mixed-version rolling deploys depend on this: a worker
+ running 6b-old code without the link task registered would
+ fail with ``KeyError`` on the link's task name, which is a
+ loud failure (no silent drop)."""
+ assert barrier_decr_and_check.name == "barrier_decr_and_check"
+
+ def test_link_max_retries_zero_prevents_double_decrement(self):
+ """The link task MUST have ``max_retries=0``. The Lua script
+ is atomic at the Redis level but the link itself is not
+ idempotent: a Celery retry would replay the script and
+ double-RPUSH + double-DECR, corrupting the aggregation
+ (counter hits 0 prematurely; results list has duplicates).
+ TTL-driven cleanup is the safety net for transient failures
+ rather than Celery autoretry."""
+ # Celery exposes ``max_retries`` on the task class.
+ assert barrier_decr_and_check.max_retries == 0
+
+ @patch("queue_backend.redis_barrier._get_redis_client")
+ def test_negative_remaining_does_not_fire_callback(self, get_redis):
+ """The Lua script returns ``remaining < 0`` when the counter
+ was TTL-expired or already cleaned up. The Python side must
+ NOT fire the callback in that branch — otherwise every
+ subsequent task completion after a TTL expiry would dispatch
+ a spurious callback."""
+ redis_client = MagicMock(name="redis")
+ # Lua returns ``[-1, []]`` — the ``< 0`` branch where keys
+ # were already DEL'd (or never existed).
+ script = MagicMock(name="script", return_value=[-1, []])
+ redis_client.register_script.return_value = script
+ get_redis.return_value = redis_client
+
+ with patch("celery.current_app") as current_app:
+ result = barrier_decr_and_check.run(
+ {"x": 1},
+ execution_id="exec-1",
+ callback_descriptor={
+ "task_name": "cb",
+ "kwargs": {"execution_id": "exec-1"},
+ "queue": "q",
+ "fairness_headers": None,
+ },
+ )
+
+ # No callback dispatch — the abandoned branch fires.
+ current_app.signature.assert_not_called()
+ assert result["status"] == "abandoned"
+ assert result["remaining"] == -1
+
+ @patch("queue_backend.redis_barrier._get_redis_client")
+ def test_callback_dispatch_failure_preserves_keys_for_ttl(self, get_redis):
+ """If ``apply_async`` raises after the Lua ``== 0`` branch
+ returned, the barrier keys MUST remain in Redis (with their
+ TTL still active) so the execution isn't stranded with no
+ recovery path.
+
+ Without deferred DEL, the previous implementation DEL'd both
+ keys inside the Lua script before Python's ``apply_async``
+ ran. A dispatch failure (broker outage, serialisation error,
+ routing failure) then left the execution with: no keys, no
+ TTL, no Celery retry (``max_retries=0``), no link_error
+ fallback — strictly worse than the chord baseline.
+
+ The fix: Lua's ``== 0`` branch returns the LRANGE results
+ WITHOUT DELing. Python DELs both keys only after
+ ``apply_async`` succeeds. A failure raises and the TTL
+ eventually cleans up the keys."""
+ redis_client = MagicMock(name="redis")
+ # ``== 0`` branch — script returns one result; Lua no longer DELs.
+ script = MagicMock(name="script", return_value=[0, [json.dumps({"x": 1})]])
+ redis_client.register_script.return_value = script
+ get_redis.return_value = redis_client
+
+ with patch("celery.current_app") as current_app:
+ cb_sig = MagicMock(name="callback_signature")
+ # apply_async raises — broker outage, serialisation error, etc.
+ cb_sig.apply_async.side_effect = RuntimeError("broker is down")
+ current_app.signature.return_value = cb_sig
+
+ with pytest.raises(RuntimeError, match=r"broker is down"):
+ barrier_decr_and_check.run(
+ {"x": 1},
+ execution_id="exec-1",
+ callback_descriptor={
+ "task_name": "cb",
+ "kwargs": {"execution_id": "exec-1"},
+ "queue": "q",
+ "fairness_headers": None,
+ },
+ )
+
+ # DELETE MUST NOT have been called — the keys must remain in
+ # Redis (with their existing TTL) so:
+ # - Late-arriving link tasks (if any) cleanly hit the Lua
+ # ``< 0`` branch and exit via the abandoned path.
+ # - TTL eventually cleans the keys.
+ # - The failure is visible via Celery's standard task-failure
+ # channels rather than a silent infinite hang.
+ redis_client.delete.assert_not_called()
+
+
+# --- TTL env validation ---
+
+
+class TestTtlEnvValidation:
+ """Misconfigured ``WORKER_BARRIER_KEY_TTL_SECONDS`` must raise.
+
+ Matches the posture of ``get_barrier()`` on a typo'd
+ ``WORKER_BARRIER_BACKEND``: a misconfigured TTL shorter than
+ execution wall-clock is a correctness issue (spurious callback
+ fires per the Lua ``< 0`` branch), so operators get a loud
+ on-startup signal rather than silent degradation.
+ """
+
+ def test_non_integer_raises(self, barrier, redis_mock, monkeypatch):
+ monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "not-an-int")
+ with pytest.raises(ValueError, match=r"not an integer"):
+ barrier.enqueue(
+ [MagicMock()],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+
+ def test_zero_raises(self, barrier, redis_mock, monkeypatch):
+ monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "0")
+ with pytest.raises(ValueError, match=r"positive integer"):
+ barrier.enqueue(
+ [MagicMock()],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+
+ def test_negative_raises(self, barrier, redis_mock, monkeypatch):
+ monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "-1")
+ with pytest.raises(ValueError, match=r"positive integer"):
+ barrier.enqueue(
+ [MagicMock()],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+
+
+# --- Mid-loop dispatch failure ---
+
+
+class TestDispatchFailureCleanup:
+ """If ``apply_async`` raises mid-fan-out, partially-dispatched
+ tasks' links would otherwise corrupt the counter and the orphan
+ state would leak. The barrier must DEL its keys before re-raising
+ so the in-flight links' DECR returns negative → Lua ``< 0``
+ branch cleanly abandons the execution.
+ """
+
+ def test_mid_loop_dispatch_failure_cleans_up_keys(
+ self, barrier, redis_mock
+ ):
+ """Task K of N raises during ``apply_async`` — barrier must
+ DEL both keys before propagating the exception."""
+ h1 = MagicMock(name="h1")
+ h2 = MagicMock(name="h2")
+ h3_failing = MagicMock(name="h3_failing")
+ h3_failing.clone.return_value.apply_async.side_effect = (
+ RuntimeError("broker outage")
+ )
+
+ with pytest.raises(RuntimeError, match=r"broker outage"):
+ barrier.enqueue(
+ [h1, h2, h3_failing],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+
+ # Initial SET + EXPIRE happened, then DEL on cleanup.
+ # The DEL call should reference BOTH barrier keys.
+ delete_calls = redis_mock.delete.call_args_list
+ # At least one of the DEL calls must include both barrier keys
+ # (the cleanup call), distinguishable from the initial
+ # ``delete(_results_key(...))`` stale-list clear.
+ cleanup_args = [
+ set(call.args) for call in delete_calls
+ if len(call.args) == 2
+ ]
+ assert any(
+ {_remaining_key("exec-1"), _results_key("exec-1")} == args
+ for args in cleanup_args
+ ), (
+ f"Expected DEL(remaining, results) in cleanup; saw "
+ f"delete calls: {[call.args for call in delete_calls]}"
+ )
+
+
+# --- link_error: barrier_abort ---
+
+
+class TestBarrierAbortLinkError:
+ """``link_error`` propagates header-task failures.
+
+ Without this, a failed header task would leave the counter stuck
+ above 0 and the execution would hang until TTL. ``barrier_abort``
+ DELs the barrier keys + logs the error; the outer task's error
+ handler is responsible for the workflow status update.
+ """
+
+ def test_abort_registered_under_canonical_name(self):
+ """Mixed-version rolling deploys require the link_error task
+ to be importable by name on every worker that processes it."""
+ from queue_backend import barrier_abort
+
+ assert barrier_abort.name == "barrier_abort"
+
+ def test_link_error_attached_to_each_header_task(self, barrier, redis_mock):
+ """Every header task must get ``barrier_abort.s(...)`` as a
+ ``.link_error``. Without this, header failures don't
+ propagate and the execution hangs."""
+ h1 = MagicMock(name="h1")
+ h2 = MagicMock(name="h2")
+ barrier.enqueue(
+ [h1, h2],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+ h1.clone.return_value.link_error.assert_called_once()
+ h2.clone.return_value.link_error.assert_called_once()
+
+ @patch("queue_backend.redis_barrier._get_redis_client")
+ def test_abort_deletes_both_barrier_keys(self, get_redis):
+ """``barrier_abort`` cleans up the barrier state on header
+ failure — DELs both keys. The first call to ``barrier_abort``
+ for a given execution_id wins the SET NX race and proceeds
+ with cleanup + log."""
+ from queue_backend import barrier_abort
+
+ redis_client = MagicMock(name="redis")
+ # ``set(...nx=True)`` returns True on first SET — this abort wins.
+ redis_client.set.return_value = True
+ get_redis.return_value = redis_client
+
+ barrier_abort.run(
+ MagicMock(name="request"),
+ RuntimeError("header task crashed"),
+ "traceback string",
+ execution_id="exec-42",
+ )
+
+ redis_client.delete.assert_called_once_with(
+ _remaining_key("exec-42"),
+ _results_key("exec-42"),
+ )
+
+ @patch("queue_backend.redis_barrier._get_redis_client")
+ def test_concurrent_aborts_deduplicate(self, get_redis):
+ """N concurrent header-task failures attach the same
+ ``link_error`` → N ``barrier_abort`` executions for one
+ logical execution failure. Without dedup that would produce
+ N ``logger.error`` calls and N Sentry events. The SET NX
+ lock makes the first abort win; subsequent aborts early-exit
+ silently."""
+ from queue_backend import barrier_abort
+
+ redis_client = MagicMock(name="redis")
+ # First call: SET NX returns True (first abort wins).
+ # Second call: SET NX returns False (lock already exists).
+ redis_client.set.side_effect = [True, False]
+ get_redis.return_value = redis_client
+
+ result_1 = barrier_abort.run(
+ MagicMock(name="request"),
+ RuntimeError("header task 1 crashed"),
+ "traceback string",
+ execution_id="exec-42",
+ )
+ result_2 = barrier_abort.run(
+ MagicMock(name="request"),
+ RuntimeError("header task 2 crashed"),
+ "traceback string",
+ execution_id="exec-42",
+ )
+
+ # First abort completed cleanup; second deduplicated.
+ assert result_1["status"] == "aborted"
+ assert result_2["status"] == "deduplicated"
+
+ # DELETE called exactly ONCE — only the winning abort cleans
+ # up. The deduplicated path early-exits before DELETE.
+ redis_client.delete.assert_called_once_with(
+ _remaining_key("exec-42"),
+ _results_key("exec-42"),
+ )
+
+ @patch("queue_backend.redis_barrier._get_redis_client")
+ def test_abort_uses_setnx_with_ttl(self, get_redis):
+ """The SET NX lock has an EX (TTL) — the lock key auto-
+ cleans alongside the barrier keys instead of leaking
+ forever on a failed execution."""
+ from queue_backend import barrier_abort
+ from queue_backend.redis_barrier import _KEY_TTL_DEFAULT_SECONDS
+
+ redis_client = MagicMock(name="redis")
+ redis_client.set.return_value = True
+ get_redis.return_value = redis_client
+
+ barrier_abort.run(
+ MagicMock(name="request"),
+ RuntimeError("crash"),
+ "traceback",
+ execution_id="exec-1",
+ )
+
+ # SET called with nx=True + ex=<ttl>.
+ redis_client.set.assert_called_once()
+ call = redis_client.set.call_args
+ assert call.args[0] == "barrier:abort_lock:exec-1"
+ assert call.kwargs.get("nx") is True
+ assert call.kwargs.get("ex") == _KEY_TTL_DEFAULT_SECONDS
+
+ @patch("queue_backend.redis_barrier._get_redis_client")
+ def test_abort_delete_failure_releases_lock(self, get_redis):
+ """If SET NX succeeds (this abort wins) but the subsequent
+ DELETE then raises (mid-abort Redis blip), the lock MUST be
+ released — otherwise sibling aborts hit the dedup early-exit
+ while the barrier keys are still alive, in-flight successful
+ tasks DECR to 0, and the callback fires with partial results
+ (silent-success-on-mid-abort-failure).
+
+ Same correctness class as the stale-lock-on-retry bug,
+ reached via a different path: mid-abort failure instead of
+ cross-execution stale state.
+ """
+ from queue_backend import barrier_abort
+ from queue_backend.redis_barrier import _abort_lock_key
+
+ redis_client = MagicMock(name="redis")
+ # SET NX returns True (this abort wins), DELETE then raises.
+ redis_client.set.return_value = True
+ # Two delete calls expected: the barrier-keys DEL that raises,
+ # then the lock-release DEL.
+ redis_client.delete.side_effect = [
+ ConnectionError("redis blip mid-abort"), # first DELETE raises
+ 1, # lock-release DEL succeeds (returns count of deleted keys)
+ ]
+ get_redis.return_value = redis_client
+
+ with pytest.raises(ConnectionError, match=r"redis blip mid-abort"):
+ barrier_abort.run(
+ MagicMock(name="request"),
+ RuntimeError("header task crashed"),
+ "traceback",
+ execution_id="exec-1",
+ )
+
+ # First DELETE was the barrier-keys cleanup; SECOND DELETE was
+ # the lock release (so siblings can re-acquire).
+ delete_calls = redis_client.delete.call_args_list
+ assert len(delete_calls) == 2, (
+ f"Expected 2 delete calls (barrier keys + lock release); "
+ f"got {len(delete_calls)}: {delete_calls}"
+ )
+ # The second call must target the abort_lock specifically.
+ assert delete_calls[1].args == (_abort_lock_key("exec-1"),), (
+ f"Expected lock-release DELETE; got args={delete_calls[1].args!r}"
+ )
+
+
+# --- Retry with reused execution_id: stale abort_lock cleanup ---
+
+
+class TestAbortLockRetryCleanup:
+ """A retry that reuses ``execution_id`` must not be poisoned by a
+ stale ``abort_lock`` left behind by the previous failed run.
+
+ Failure mode without the fix: execution A fails → ``barrier_abort``
+ writes ``abort_lock:foo`` (TTL 6h) → within 6h, execution B retries
+ with ``exec_id=foo`` → ``enqueue`` resets remaining/results but
+ abort_lock survives → if any header in B fails, its ``barrier_abort``
+ hits the stale lock and early-exits without DELing B's keys →
+ in-flight successful tasks DECR ``remaining`` to 0 → callback fires
+ with partial results → failed retry silently masked as successful.
+
+ Fix: ``enqueue`` DELs the abort_lock alongside the results list at
+ setup time, so a retry starts with a clean slate.
+ """
+
+ def test_enqueue_clears_stale_abort_lock(self, barrier, redis_mock):
+ """``enqueue`` must DEL the abort_lock key at setup so a
+ stale lock from a previous (failed) execution with the same
+ execution_id doesn't poison the retry's abort path."""
+ from queue_backend.redis_barrier import _abort_lock_key
+
+ barrier.enqueue(
+ [MagicMock(name="h1")],
+ callback_task_name="cb",
+ callback_kwargs={"execution_id": "exec-foo"},
+ callback_queue="q",
+ app_instance=MagicMock(),
+ )
+ # The setup DEL must include the abort_lock key (alongside
+ # results) — otherwise a stale lock from a prior run with the
+ # same execution_id survives into this enqueue.
+ redis_mock.delete.assert_called_once_with(
+ _results_key("exec-foo"),
+ _abort_lock_key("exec-foo"),
+ )
+
+
+# --- Process-local Redis client caching ---
+
+
+class TestRedisClientSingleton:
+ """``_get_redis_client()`` returns a process-cached client so the
+ high-frequency ``barrier_decr_and_check`` / ``barrier_abort``
+ invocations reuse the same ``ConnectionPool`` rather than spinning
+ up a fresh pool (and TCP handshakes) per task. At 1000-task
+ fan-out the difference is 1000 vs 1 pool initialisation.
+ """
+
+ def test_get_redis_client_returns_same_instance(self):
+ """Repeated calls within one process must return the same
+ ``Redis`` instance — guarantees connection-pool reuse."""
+ from queue_backend import redis_barrier
+
+ # Reset the cache so this test exercises the lazy-init path.
+ redis_barrier._redis_client_singleton = None
+ try:
+ with patch(
+ "queue_backend.redis_barrier.create_redis_client"
+ ) as factory:
+ factory.return_value = MagicMock(name="redis_client")
+ client_a = redis_barrier._get_redis_client()
+ client_b = redis_barrier._get_redis_client()
+ client_c = redis_barrier._get_redis_client()
+
+ # All three calls return the same object.
+ assert client_a is client_b is client_c
+ # ``create_redis_client`` factory called exactly ONCE —
+ # subsequent calls hit the cache.
+ assert factory.call_count == 1
+ finally:
+ # Reset for downstream tests.
+ redis_barrier._redis_client_singleton = None
+
+ def test_falls_back_to_canonical_redis_prefix_for_sentinel_ssl_inheritance(
+ self, monkeypatch
+ ):
+ """When no ``WORKER_BARRIER_REDIS_HOST`` is set, the barrier
+ MUST use the canonical ``REDIS_`` prefix directly so it
+ inherits the FULL config including ``REDIS_SENTINEL_MODE`` and
+ ``REDIS_SSL`` — not just HOST/PORT/PASSWORD/USER/DB.
+
+ Without this fix, a Sentinel-backed deployment that relies on
+ the documented fallback would connect standalone (Sentinel
+ mode is per-prefix in ``create_redis_client`` and doesn't
+ cross-fall-back), failing every execution at flag-flip time."""
+ from queue_backend import redis_barrier
+
+ # Ensure no barrier-specific HOST is set.
+ monkeypatch.delenv("WORKER_BARRIER_REDIS_HOST", raising=False)
+ redis_barrier._redis_client_singleton = None
+ try:
+ with patch(
+ "queue_backend.redis_barrier.create_redis_client"
+ ) as factory:
+ factory.return_value = MagicMock(name="redis_client")
+ redis_barrier._get_redis_client()
+
+ # The factory MUST be called with env_prefix="REDIS_" so
+ # SENTINEL_MODE/SSL inherit from the canonical instance.
+ factory.assert_called_once()
+ assert factory.call_args.kwargs.get("env_prefix") == "REDIS_", (
+ f"Expected env_prefix='REDIS_' when no barrier-specific "
+ f"HOST is set (so Sentinel/SSL inherit); got "
+ f"env_prefix={factory.call_args.kwargs.get('env_prefix')!r}"
+ )
+ finally:
+ redis_barrier._redis_client_singleton = None
+
+ def test_uses_barrier_prefix_when_dedicated_host_set(self, monkeypatch):
+ """When ``WORKER_BARRIER_REDIS_HOST`` IS set, the operator has
+ opted into a dedicated barrier Redis and is responsible for
+ the full barrier-prefix config (HOST + PORT + ... +
+ SENTINEL_MODE + SSL)."""
+ from queue_backend import redis_barrier
+
+ monkeypatch.setenv("WORKER_BARRIER_REDIS_HOST", "dedicated-redis")
+ redis_barrier._redis_client_singleton = None
+ try:
+ with patch(
+ "queue_backend.redis_barrier.create_redis_client"
+ ) as factory:
+ factory.return_value = MagicMock(name="redis_client")
+ redis_barrier._get_redis_client()
+
+ assert (
+ factory.call_args.kwargs.get("env_prefix")
+ == "WORKER_BARRIER_REDIS_"
+ ), (
+ f"Expected env_prefix='WORKER_BARRIER_REDIS_' when "
+ f"dedicated HOST is set; got "
+ f"env_prefix={factory.call_args.kwargs.get('env_prefix')!r}"
+ )
+ finally:
+ redis_barrier._redis_client_singleton = None
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])