LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3523 [FEAT] PG Queue Phase 6b — RedisDecrBarrier + WORKER_BARRIER_BACKEND flag (#2026)
ali77d09d5a2e09
diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.pyindex a56f0eb7c..f4bec94a7 100644--- a/workers/api-deployment/tasks.py+++ b/workers/api-deployment/tasks.py@@ -696,20 +696,11 @@ def _run_workflow_api(fairness=api_fairness,)- if not result:- # TODO(phase-6b): tighten to ``if result is None:``- # when ``RedisDecrBarrier`` lands. The ``Barrier`` Protocol- # declares ``None`` as the *sole* "no work enqueued" signal,- # so identity-against-None is the principled check. Safe to- # defer today because Celery's ``AsyncResult`` has no custom- # ``__bool__`` (always truthy), making the truthiness and- # identity checks behaviourally equivalent. A future- # substrate handle with a falsy-but-not-None ``__bool__``- # would be misrouted into the zero-files path under the- # current truthiness check — that's the regression this TODO- # forecloses, alongside the substrate that motivates it.- #- # Two reasons ``result`` can be falsy:+ if result is None:+ # Identity-against-``None`` mirrors the ``Barrier`` Protocol+ # contract verbatim: ``None`` is the *sole* signal of "no+ # work enqueued" (substrate-level failures raise instead).+ # Two reasons ``result`` can be ``None``:# 1. Zero ``batch_tasks`` — the barrier short-circuits and# returns ``None``. Pre-Barrier this would have called# ``chord(empty)(callback)`` which Celery handles bydiff --git a/workers/queue_backend/__init__.py b/workers/queue_backend/__init__.pyindex 2e7395135..3fcb1bab6 100644--- a/workers/queue_backend/__init__.py+++ b/workers/queue_backend/__init__.py@@ -2,18 +2,103 @@Single place where the substrate choice (Celery today; PG Queue later)lives. Both entry points are transparent passthroughs to Celery today.++The ``Barrier`` substrate is runtime-selectable via the+``WORKER_BARRIER_BACKEND`` env var:++- ``chord`` (default) — ``CeleryChordBarrier``, the thin wrapper around+ ``celery.chord(header)(body)``. Byte-identical wire to the+ pre-abstraction code path; current production behaviour.+- ``redis`` — ``RedisDecrBarrier``, the labs-design distributed-counter+ pattern (``DECR remaining:{exec_id}`` + per-task ``RPUSH``-based+ result aggregation). Replaces only the chord aggregation primitive;+ dispatch / retries / monitoring still ride on Celery.++**Default-safety posture.** The env defaults to ``chord``, so a merge+into ``main`` is zero-impact: a downstream env (``unstract-cloud``,+self-hosted, etc.) that does not set ``WORKER_BARRIER_BACKEND`` runs+the same substrate it ran yesterday. The new ``RedisDecrBarrier`` code+is dormant until an operator explicitly flips the flag.++Unknown values raise — operators get a loud error at module import+time rather than silently falling back, which would mask a typo'd+flag in a production env."""+import os+from enum import StrEnum+from .barrier import Barrier, BarrierHandle, CeleryChordBarrierfrom .decorator import worker_taskfrom .dispatch import dispatchfrom .fairness import FairnessKey+from .redis_barrier import (+ RedisDecrBarrier,+ barrier_abort,+ barrier_decr_and_check,+)__all__ = ["Barrier",+ "BarrierBackend","BarrierHandle","CeleryChordBarrier","FairnessKey",+ "RedisDecrBarrier",+ "barrier_abort",+ "barrier_decr_and_check","dispatch",+ "get_barrier","worker_task",]+++class BarrierBackend(StrEnum):+ """Enumeration of ``Barrier`` substrate choices.++ Used as the canonical reference for the ``WORKER_BARRIER_BACKEND``+ env value, the factory's dispatch table, and tests that flip the+ env. Subclassing ``StrEnum`` (Python 3.11+) means ``BarrierBackend.CHORD+ == "chord"`` at runtime, so the enum members can stand in anywhere+ a string is expected — including ``os.environ`` reads.+ """++ CHORD = "chord"+ REDIS = "redis"+++def get_barrier() -> Barrier:+ """Return the ``Barrier`` implementation selected by env.++ Reads ``WORKER_BARRIER_BACKEND`` at call time so test harnesses+ that ``monkeypatch.setenv`` can flip the substrate per-test+ without a module reload. Production call sites construct the+ barrier once per process (module-level singleton in+ ``orchestration_utils.py``) so the env read happens at worker+ startup — flag flips require pod restart, same posture as every+ other ``WORKER_*`` env in the codebase.++ Default ``BarrierBackend.CHORD`` mirrors the current production+ substrate. An unknown value raises ``ValueError`` so a typo'd+ ``redis`` → ``rediz`` doesn't silently fall back to chord (which+ would invalidate the canary).+ """+ raw = os.getenv("WORKER_BARRIER_BACKEND", BarrierBackend.CHORD.value)+ try:+ backend = BarrierBackend(raw.strip().lower())+ except ValueError:+ valid = tuple(b.value for b in BarrierBackend)+ raise ValueError(+ f"WORKER_BARRIER_BACKEND={raw!r} is not a recognised "+ f"barrier backend. Valid values: {valid}. Unset the env var "+ f"to default to {BarrierBackend.CHORD.value!r}."+ ) from None++ if backend is BarrierBackend.CHORD:+ return CeleryChordBarrier()+ if backend is BarrierBackend.REDIS:+ return RedisDecrBarrier()+ # Unreachable — StrEnum constructor would have raised above for+ # anything not in the enum. Defensive raise so the type checker+ # sees an exhaustive match.+ raise AssertionError(f"Unhandled BarrierBackend: {backend!r}")diff --git a/workers/queue_backend/redis_barrier.py b/workers/queue_backend/redis_barrier.pynew file mode 100644index 000000000..5f79db9cd--- /dev/null+++ b/workers/queue_backend/redis_barrier.py@@ -0,0 +1,803 @@+"""Redis-DECR Barrier — alternative substrate to ``CeleryChordBarrier``.++Implements the ``Barrier`` Protocol via the labs-design distributed-counter+pattern (``DECR remaining:{exec_id}``) with per-task result aggregation+in a Redis list. Selected at runtime by the ``WORKER_BARRIER_BACKEND``+env flag (default ``chord`` — see ``queue_backend.get_barrier``).++**Why this substrate exists.** Celery's ``chord(header)(body)`` aggregator+is the highest-risk Celery construct at our scale — documented silent+task drops at ~130K-task scale (see PG Queue decision journey). This+substrate replaces *only* the aggregation primitive; everything else+(dispatch, retries, result backend, monitoring) still rides on Celery.+The transport is unchanged — only the "wait for N tasks to complete,+then fire the callback with their results" coordination is moved from+Celery's chord backend to Redis.++**Wire model.**++1. ``enqueue``: ``SET remaining:{exec_id} N`` with TTL (default 6h,+ tunable via ``WORKER_BARRIER_KEY_TTL_SECONDS``). The results list+ is created lazily by the first link task's ``RPUSH`` — its TTL is+ set inside the Lua script after the push, so both keys share the+ same expiry window. Each header task is dispatched with+ ``.link(barrier_decr_and_check.s(...))`` (success) and+ ``.link_error(barrier_abort.s(...))`` (failure).+2. Per-task success: the link task runs ``RPUSH results + EXPIRE ++ DECR remaining`` atomically via a Lua script. If the post-decrement+ counter reads exactly 0, the link reads the full results list,+ dispatches the callback with it as the first arg, then deletes+ the Redis keys.+3. Per-task failure: ``barrier_abort`` runs as a link_error, DELing+ both barrier keys + logging the failure. Mirrors Celery chord's+ default error-propagation semantic (callback isn't invoked when+ any header fails). The outer task's error handler marks the+ workflow FAILED; this task only cleans barrier state.+4. TTL safety net: if the link/link_error never fire for some task+ (worker crash mid-execution, broker outage), TTL expires the keys+ after the configured window. Subsequent successful tasks' DECR on+ the missing counter returns negative; the Lua script's ``< 0``+ branch DELs cleanly without firing a spurious callback.++**Result-aggregation parity.** The callback receives+``list[BatchExecutionResult]`` exactly as it does today under+``CeleryChordBarrier`` — zero callback-side changes. Serialisation+rides through ``BatchExecutionResult.to_dict()`` / ``from_dict()``+(the typed callback boundary) ↔ JSON in Redis.++**Atomicity.** The per-task ``RPUSH + DECR`` is atomic via Lua — a+network partition mid-step can't split-brain the counter and the+result list. ``DECR`` is also atomic on its own, so exactly one task+sees the post-decrement 0.++**Mixed-version rolling deploy.** The ``WORKER_BARRIER_BACKEND`` env+var is read at worker startup, so each pod independently picks its+substrate. A given execution's tasks all share the publishing pod's+substrate choice (the barrier is selected once per execution start);+no cross-substrate aggregation within a single execution.+"""++from __future__ import annotations++import contextlib+import json+import logging+import os+from dataclasses import dataclass+from typing import TYPE_CHECKING, Any, TypedDict++from celery.canvas import Signature++from unstract.core.cache.redis_client import create_redis_client++from .decorator import worker_task+from .fairness import FairnessKey+from .handle import BarrierHandle++if TYPE_CHECKING:+ import redis as redis_lib++logger = logging.getLogger(__name__)+++# Env-var prefix for the barrier's Redis connection. Falls back to+# ``REDIS_`` (the canonical worker Redis instance) via ``create_redis_client``'s+# nested-getenv pattern. Operators can point the barrier at a separate+# Redis by setting ``WORKER_BARRIER_REDIS_HOST`` / ``_PORT`` etc.+_BARRIER_REDIS_ENV_PREFIX = "WORKER_BARRIER_REDIS_"++# Belt-and-suspenders cleanup for orphaned ``remaining`` / ``results``+# keys. On the happy path the Lua script ``DEL``s both keys when+# ``remaining`` reaches 0 — TTL only matters when the counter stalls+# (e.g. some header tasks fail and the link never fires for them).+#+# **Must be strictly > longest plausible execution wall-clock.** If+# TTL expires while tasks are still pending, the next successful+# task's ``DECR`` creates the counter at ``-1`` and the script fires+# the callback with just that task's result — followed by more+# spurious callbacks as remaining tasks complete.+#+# Reference budget: the operator-tuned ceiling is whatever+# ``FILE_PROCESSING_TASK_TIME_LIMIT`` is set to in the deployment env+# (workers/sample.env ships 7200/2h; docker/sample.env up to 10800/3h).+# ``process_file_batch`` has ``max_retries=0`` so no retry amplification —+# worst case per batch is the time-limit ceiling. Multi-batch+# executions ≈ max(per-batch time) since batches run in parallel+# (worker concurrency is the limiter, not serial wait).+#+# 6h default gives 2-3x margin over the documented per-batch ceiling+# while staying in the same order of magnitude as+# ``FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000`` (5h). Operators with+# longer workflows (e.g. multi-step pipelines with chained barriers)+# or shorter known max-execution-time should tune via+# ``WORKER_BARRIER_KEY_TTL_SECONDS``.+_KEY_TTL_DEFAULT_SECONDS = 6 * 60 * 60 # 6h+++def _key_ttl_seconds() -> int:+ """Read the TTL from env, applying the default only on absence.++ Read at call time (not module import) so a test+ ``monkeypatch.setenv`` flips the value without a module reload.++ Invalid values (non-int, negative, zero) **raise**, matching the+ posture in ``get_barrier()`` where ``WORKER_BARRIER_BACKEND=rediz``+ raises rather than silently falling back to chord. A misconfigured+ TTL shorter than execution wall-clock is a correctness issue per+ this file's docstring (would cause spurious callback fires) —+ operators get the same loud-on-misconfig signal as for the+ backend flag.+ """+ raw = os.getenv("WORKER_BARRIER_KEY_TTL_SECONDS")+ if raw is None:+ return _KEY_TTL_DEFAULT_SECONDS+ try:+ value = int(raw)+ except ValueError as e:+ raise ValueError(+ f"WORKER_BARRIER_KEY_TTL_SECONDS={raw!r} is not an integer. "+ f"Unset the env var to default to {_KEY_TTL_DEFAULT_SECONDS}s "+ f"(6h)."+ ) from e+ if value <= 0:+ raise ValueError(+ f"WORKER_BARRIER_KEY_TTL_SECONDS={value} must be a positive "+ f"integer. Unset the env var to default to "+ f"{_KEY_TTL_DEFAULT_SECONDS}s (6h)."+ )+ return value+++# Atomic ``RPUSH + EXPIRE + DECR``: returns ``(remaining, results)``.+#+# Three branches the Python side reads off the returned counter:+#+# remaining > 0 — pending, no results returned (still aggregating)+# remaining == 0 — complete, all results returned (we are the last).+# Keys are NOT DEL'd here — Python defers the DEL+# until after ``apply_async()`` succeeds, so that+# a callback-dispatch failure leaves the keys (and+# their TTL) in place rather than stranding the+# execution with no state + no recovery path.+# remaining < 0 — TTL-expired counter (or replay after cleanup);+# keys DEL'd inside the script to prevent further+# spurious fires, no callback dispatched.+#+# Setting the ``results`` key TTL inside the script — ``EXPIRE`` on+# the call AFTER ``RPUSH`` ensures the key exists when EXPIRE runs+# (issuing EXPIRE before any push is a no-op on a non-existent key,+# which is the bug the previous version had). Repeated EXPIRE per+# RPUSH is cheap and keeps both keys' TTLs aligned.+_RPUSH_DECR_LUA = """+local remaining_key = KEYS[1]+local results_key = KEYS[2]+local result_json = ARGV[1]+local ttl_seconds = tonumber(ARGV[2])+redis.call("RPUSH", results_key, result_json)+redis.call("EXPIRE", results_key, ttl_seconds)+local remaining = redis.call("DECR", remaining_key)+if remaining == 0 then+ -- We're the last task. LRANGE the results for the Python side to+ -- dispatch the callback. DO NOT DEL the keys here — Python defers+ -- the DEL until after apply_async() succeeds so a dispatch failure+ -- (broker outage, serialisation error) leaves the keys + TTL in+ -- place. Without this deferral, an apply_async failure would+ -- strand the execution with both keys gone, no TTL, no Celery+ -- retry (max_retries=0), and no link_error fallback — strictly+ -- worse than the chord baseline (where Celery's chord backend+ -- owns + retries body invocation).+ local all_results = redis.call("LRANGE", results_key, 0, -1)+ return {remaining, all_results}+end+if remaining < 0 then+ -- Counter was TTL-expired (or already cleaned up) when this task+ -- ran. ``DECR`` on a missing key created it at ``-1``; subsequent+ -- tasks would land here too. DEL both keys so neither this task+ -- nor any subsequent one fires a spurious callback; the abandoned+ -- branch on the Python side surfaces an error-severity log.+ redis.call("DEL", remaining_key, results_key)+ return {remaining, {}}+end+return {remaining, {}}+"""+++# Process-local cached Redis client. Lazy-initialised on first call+# within a worker process; reused across every ``barrier_decr_and_check``+# / ``barrier_abort`` / ``RedisDecrBarrier`` invocation in that process.+#+# Why a singleton: ``create_redis_client`` builds a fresh+# ``ConnectionPool`` each call. Without caching, a 1000-task fan-out+# would spin up 1000 separate pools (1000 TCP handshakes torn down in+# sequence) — exactly the scale this substrate exists to handle.+#+# Fork safety: Celery's prefork model re-imports this module in each+# child process, so each worker process gets its own fresh+# ``_redis_client_singleton`` — no socket sharing across fork+# boundaries (which would corrupt state).+#+# Thread safety: redis-py's ``ConnectionPool`` is internally thread-safe,+# so concurrent users of the cached client are fine. The lazy-init+# itself is not locked — two threads racing on the first call could+# briefly create two pools, with one being orphaned. The cost is one+# short-lived pool (immediately GC'd), not a correctness issue.+_redis_client_singleton: redis_lib.Redis | None = None+++def _get_redis_client() -> redis_lib.Redis:+ """Return a process-cached Redis client for the barrier.++ ``create_redis_client``'s nested-getenv fallback only covers+ HOST/PORT/PASSWORD/USER/DB — it does NOT cross-fall-back+ SENTINEL_MODE or SSL. In a deployment where the canonical Redis+ is Sentinel-backed or TLS-secured, leaving ``WORKER_BARRIER_REDIS_*``+ unset and relying on the documented HOST fallback would result in+ the barrier inheriting the right host/port but connecting+ standalone/plaintext — which fails (or worse, connects without+ TLS) every execution the moment the flag flips.++ To avoid that footgun: when no barrier-specific HOST is set, use+ the canonical ``REDIS_`` prefix *directly* so we inherit the FULL+ canonical config (including Sentinel + SSL). When an operator+ has set ``WORKER_BARRIER_REDIS_HOST`` they've opted into a+ dedicated barrier Redis and are responsible for the full config+ (HOST + PORT + ... + SENTINEL_MODE + SSL).++ ``decode_responses=True`` so ``LRANGE`` returns ``list[str]``+ (we JSON-decode each entry).+ """+ global _redis_client_singleton+ if _redis_client_singleton is None:+ # If no dedicated barrier-Redis host is configured, use the+ # canonical REDIS_ prefix so Sentinel/SSL config inherits too.+ if os.getenv(f"{_BARRIER_REDIS_ENV_PREFIX}HOST"):+ env_prefix = _BARRIER_REDIS_ENV_PREFIX+ else:+ env_prefix = "REDIS_"+ _redis_client_singleton = create_redis_client(+ env_prefix=env_prefix,+ decode_responses=True,+ socket_timeout=5,+ socket_connect_timeout=5,+ )+ return _redis_client_singleton+++def _remaining_key(execution_id: str) -> str:+ return f"barrier:remaining:{execution_id}"+++def _results_key(execution_id: str) -> str:+ return f"barrier:results:{execution_id}"+++def _abort_lock_key(execution_id: str) -> str:+ """Per-execution dedup lock for ``barrier_abort``.++ Set via ``SET NX EX`` in ``barrier_abort`` to collapse N concurrent+ header-task failures into a single Sentry/error event. MUST be+ cleared at the start of a new barrier (in ``enqueue``) — otherwise+ a retry that reuses an ``execution_id`` would hit a stale lock,+ early-exit the abort path, and silently mask the failed retry as+ successful.+ """+ return f"barrier:abort_lock:{execution_id}"+++class RedisDecrBarrier:+ """``Barrier`` implementation via Redis ``DECR remaining`` pattern.++ Drop-in replacement for ``CeleryChordBarrier`` from the call+ sites' perspective — same ``enqueue(header_tasks, ...)`` signature,+ same ``BarrierHandle | None`` return contract, same fairness+ plumbing. The substrate difference is invisible above the+ ``Barrier`` Protocol.++ See module docstring for the wire model, atomicity, and failure+ semantics.+ """++ def __init__(self, redis_client: redis_lib.Redis | None = None) -> None:+ """Args:+ redis_client: Optional pre-built client. Tests pass a fake+ (``fakeredis``) here. Production leaves it ``None`` so+ the barrier builds its own from env on first use.+ """+ self._redis_client = redis_client++ @property+ def redis(self) -> redis_lib.Redis:+ """Lazy-init the Redis client.++ Built on first use rather than at construction time so the+ module can be imported (and the worker task registered) in+ contexts where Redis isn't yet reachable (e.g. test collection,+ worker boot before networking up).++ Production path: ``self._redis_client is None`` → falls back to+ (and caches) the module-level ``_redis_client_singleton``, so+ the instance and the standalone ``barrier_decr_and_check`` /+ ``barrier_abort`` worker tasks all share the same+ ``ConnectionPool``.++ Test path: ``__init__(redis_client=...)`` bypasses this branch+ entirely — fixtures inject a ``MagicMock`` / ``fakeredis``+ without touching the module-level singleton (which would leak+ across tests).+ """+ if self._redis_client is None:+ self._redis_client = _get_redis_client()+ return self._redis_client++ def enqueue(+ self,+ header_tasks: list[Signature],+ *,+ callback_task_name: str,+ callback_kwargs: dict[str, Any],+ callback_queue: str,+ app_instance: Any,+ fairness: FairnessKey | None = None,+ ) -> BarrierHandle | None:+ """See :class:`queue_backend.barrier.Barrier.enqueue`.++ Mirrors ``CeleryChordBarrier.enqueue`` semantics:++ - Empty ``header_tasks`` → ``None`` (caller handles the+ zero-files contract); no Redis keys touched.+ - Any substrate-level failure (Redis down, dispatch error,+ serialisation issue) → raises.+ - Success → returns a ``BarrierHandle`` whose ``.id`` is the+ execution id (no ``AsyncResult`` since there's no Celery+ chord wrapping the group; the id is what the call site+ logs for chord-id traceability).++ Note: ``app_instance`` is accepted for ``Barrier`` Protocol+ parity with ``CeleryChordBarrier`` but unused by this+ substrate. The callback signature is built inside the link+ task via ``celery.current_app.signature(...)`` (so the link+ runs against the *worker's* app, not the producer's). Keeping+ the parameter in the signature lets call sites swap between+ substrates without per-substrate adapter code.+ """+ # Explicit ``del`` documents the Protocol-parity intent and+ # satisfies static linters' unused-parameter check.+ del app_instance+ if not header_tasks:+ execution_id = callback_kwargs.get("execution_id")+ pipeline_id = callback_kwargs.get("pipeline_id")+ logger.info(+ f"[exec:{execution_id}] [pipeline:{pipeline_id}] "+ "Zero header tasks detected — skipping barrier enqueue "+ "(parent should handle pipeline status updates directly)"+ )+ return None++ execution_id = callback_kwargs.get("execution_id")+ if not execution_id:+ raise ValueError(+ "RedisDecrBarrier requires execution_id in callback_kwargs — "+ "it's the key suffix for the per-execution remaining/results "+ "Redis keys"+ )+ execution_id = str(execution_id)++ try:+ fairness_headers = fairness.as_header() if fairness else None++ # The link task body needs to know:+ # - which execution's counter to decrement+ # - how to reconstruct + dispatch the callback when count hits 0+ # All Celery-serialisable so the link can run on any worker.+ callback_descriptor: CallbackDescriptor = {+ "task_name": callback_task_name,+ "kwargs": callback_kwargs,+ "queue": callback_queue,+ "fairness_headers": fairness_headers,+ }++ # Initialise the counter before any header task can fire.+ # ``SET remaining N EX <ttl>`` overwrites any stale counter+ # left from a prior run with the same exec_id; the upfront+ # ``DEL results, abort_lock`` (below) clears the rest of+ # the prior-run state (the results list grows via RPUSH so+ # we must clear it explicitly; the abort_lock would otherwise+ # silently mask a retry's failure as success — see the+ # comment block at the DELETE call). ``ttl`` is+ # ``_key_ttl_seconds()`` (6h default).+ ttl_seconds = _key_ttl_seconds()+ # Clear any leftover state from a prior execution with this+ # execution_id (including the dedup lock written by a+ # previous run's ``barrier_abort``). Without DELing the+ # abort_lock here, a retry that reuses execution_id would+ # find the stale lock and route every ``barrier_abort``+ # straight to the deduplicated branch — leaving B's+ # ``remaining``/``results`` keys uncleaned. In-flight+ # successful tasks from B would then hit ``remaining == 0``+ # via DECR and fire the callback with partial results,+ # silently masking the failed retry as successful.+ self.redis.delete(+ _results_key(execution_id),+ _abort_lock_key(execution_id),+ )+ self.redis.set(+ _remaining_key(execution_id),+ len(header_tasks),+ ex=ttl_seconds,+ )+ # ``results`` key TTL is set inside the Lua script on+ # first RPUSH — ``EXPIRE`` on a non-existent key is a+ # no-op, so we can't pre-set it here.++ # Stamp fairness on each header task and attach the link+ # (success) + link_error (failure) callbacks.+ # ``Signature.clone()`` avoids mutating the caller's list+ # (same reasoning as ``CeleryChordBarrier``).+ link_signature = barrier_decr_and_check.s(+ execution_id=execution_id,+ callback_descriptor=callback_descriptor,+ )+ # link_error explicitly propagates header-task failures —+ # mirrors Celery chord's default error semantic (chord+ # callback isn't invoked on header failure). Without this,+ # a failed header task would leave the counter stuck and+ # the execution would hang until TTL cleanup. The abort+ # task DELs the barrier keys + logs the error; the outer+ # task's error path is responsible for the workflow+ # status update.+ link_error_signature = barrier_abort.s(execution_id=execution_id)+ for i, task in enumerate(header_tasks):+ try:+ cloned = task.clone()+ if fairness_headers:+ cloned.set(headers=fairness_headers)+ cloned.link(link_signature)+ cloned.link_error(link_error_signature)+ cloned.apply_async()+ except Exception:+ # Mid-loop dispatch failure: tasks 0..i-1 are+ # already in flight with the link attached, but+ # ``i`` of N never made it to the broker. The+ # counter would never reach 0 → execution hangs.+ # DEL the barrier keys so the in-flight links'+ # DECR-on-missing returns negative and the Lua+ # script's ``< 0`` branch DELs cleanly. Re-raise+ # so the caller's status-update path marks the+ # workflow ERROR.+ self.redis.delete(+ _remaining_key(execution_id),+ _results_key(execution_id),+ )+ # ``logger.exception`` captures the underlying+ # apply_async failure cause (broker timeout vs+ # serialisation error vs routing error) alongside+ # the orphan-count context — without it, a reader+ # of this log can't tell why dispatch failed and+ # has to cross-reference the outer handler's log.+ logger.exception(+ f"[exec:{execution_id}] apply_async failed at "+ f"task {i}/{len(header_tasks)}; {i} orphan tasks "+ f"already dispatched. Barrier keys DEL'd to "+ f"prevent spurious callback fires from the "+ f"orphan tasks' link decrements."+ )+ raise++ logger.info(+ f"Barrier enqueued via RedisDecrBarrier — "+ f"exec_id={execution_id}, "+ f"header_tasks={len(header_tasks)}, "+ f"callback={callback_task_name}, "+ f"queue={callback_queue}"+ )++ # Handle's ``.id`` is the execution id — same call sites+ # log ``chord_id`` from this; the semantic shifts from+ # "Celery chord aggregator task id" to "execution id" but+ # both are stable per-execution identifiers and the log+ # consumers don't depend on the underlying task vs exec id+ # distinction.+ return _RedisBarrierHandle(id=execution_id)++ except Exception:+ execution_id = callback_kwargs.get("execution_id")+ pipeline_id = callback_kwargs.get("pipeline_id")+ logger.exception(+ f"[exec:{execution_id}] [pipeline:{pipeline_id}] "+ f"Failed to enqueue barrier via Redis "+ f"(callback={callback_task_name}, queue={callback_queue}, "+ f"header_tasks={len(header_tasks)})"+ )+ raise+++class CallbackDescriptor(TypedDict):+ """Shape of the dict baked into the link signature and re-read on+ the worker that runs ``barrier_decr_and_check``.++ This descriptor crosses a serialisation boundary (Celery+ ``signature(...)`` → broker → consumer worker), so the four-key+ contract is otherwise enforced only by string literals duplicated+ across producer and consumer. Typing it as a ``TypedDict`` gives+ the type checker a chance to catch typos / renames before they+ surface as remote ``KeyError`` mid-aggregation.++ ``fairness_headers`` is always present in the dict; ``None`` when+ the producer passed no ``FairnessKey``.+ """++ task_name: str+ kwargs: dict[str, Any]+ queue: str+ fairness_headers: dict[str, Any] | None+++@dataclass(frozen=True, slots=True)+class _RedisBarrierHandle:+ """Minimal ``BarrierHandle`` implementation.++ Just an ``id`` attribute (the execution id) so call sites that+ log ``handle.id`` for chord-id tracing keep working. A class+ (rather than a ``namedtuple``) gives a clear home for the+ docstring + ``slots=True`` memory control; the ``BarrierHandle``+ Protocol is structural so any object exposing ``id: str`` would+ type-check.+ """++ id: str+++@worker_task(+ name="barrier_decr_and_check",+ # ``max_retries=0`` — the link task is intentionally non-idempotent+ # at the Redis-script level (RPUSH + DECR are committed atomically,+ # but a Celery retry would replay the script and double-count). Any+ # Celery-level retry would cause a second RPUSH + DECR, corrupting+ # the aggregation. If the link fails after the Lua commit (e.g. the+ # subsequent callback ``apply_async`` raises), the counter has+ # already advanced — TTL-driven cleanup is the safety net, and the+ # outer execution's error path marks the workflow status.+ #+ # Trade-off: a transient Redis blip on the link's Lua call means+ # this task's contribution is lost (counter doesn't advance) and+ # the execution hangs until TTL. Acceptable because (a) the+ # alternative — letting Celery retry — corrupts the aggregation,+ # and (b) Redis blips are operator-visible via monitoring, unlike+ # silent double-counts.+ max_retries=0,+)+def barrier_decr_and_check(+ result: Any,+ *,+ execution_id: str,+ callback_descriptor: CallbackDescriptor,+) -> dict[str, Any]:+ """Per-task link callback for ``RedisDecrBarrier``.++ Runs after each header task completes successfully. Pushes the+ task's result into the per-execution Redis list, decrements the+ counter, and (if the post-decrement counter reads 0) dispatches+ the aggregating callback with the full result list.++ ``result`` is whatever the header task returned — for ``process_file_batch``+ that's a ``BatchExecutionResult`` already serialised to a dict via+ the typed callback boundary. We pass it through ``json.dumps`` to+ survive the Redis round-trip.+ """+ from celery import current_app++ try:+ # No ``default=str`` — that would silently stringify any non-+ # JSON-safe leaf (datetime, UUID, Decimal, sets...) and mask+ # the typed-boundary regression the surrounding ``try/except``+ # is here to surface. The header task's return value is+ # expected to be a fully JSON-safe dict per the+ # ``BatchExecutionResult.to_dict()`` contract; anything else+ # should fail loudly here so the regression surfaces.+ result_json = json.dumps(result)+ except (TypeError, ValueError):+ logger.exception(+ f"[exec:{execution_id}] Header task result is not "+ f"JSON-serialisable — barrier aggregation cannot proceed. "+ f"This indicates a typed-boundary regression; the "+ f"BatchExecutionResult.to_dict() contract must produce a "+ f"JSON-safe shape."+ )+ raise++ redis_client = _get_redis_client()+ script = redis_client.register_script(_RPUSH_DECR_LUA)+ remaining, raw_results = script(+ keys=[+ _remaining_key(execution_id),+ _results_key(execution_id),+ ],+ args=[result_json, _key_ttl_seconds()],+ )++ logger.info(f"[exec:{execution_id}] Barrier link DECR → remaining={remaining}")++ if remaining > 0:+ # More tasks pending — nothing to do.+ return {"status": "pending", "remaining": remaining}++ if remaining < 0:+ # TTL-expired counter (or replay after cleanup). The Lua+ # script already DEL'd both keys to prevent further spurious+ # fires on subsequent task completions. No callback dispatch.+ #+ # Logged at ERROR (not WARNING) because reaching this branch+ # means the barrier was torn down out from under in-flight+ # tasks — by definition an abnormal terminal state where the+ # execution almost certainly did not complete normally.+ # Returning normally from a .link is indistinguishable from+ # success to Celery; the ERROR log is the only execution-id-+ # tagged signal an operator can correlate to the hung/+ # incorrectly-statused execution.+ logger.error(+ f"[exec:{execution_id}] Barrier abandoned — counter went "+ f"negative (remaining={remaining}); keys were TTL-expired "+ f"or torn down. No callback dispatched; execution likely "+ f"in an inconsistent terminal state and needs investigation."+ )+ return {"status": "abandoned", "remaining": remaining}++ # remaining == 0: we're the last task. Dispatch the callback.+ all_results = [json.loads(r) for r in raw_results]+ callback_task_name = callback_descriptor["task_name"]+ callback_kwargs = callback_descriptor["kwargs"]+ callback_queue = callback_descriptor["queue"]+ fairness_headers = callback_descriptor.get("fairness_headers")++ callback_signature = current_app.signature(+ callback_task_name,+ args=[all_results],+ kwargs=callback_kwargs,+ queue=callback_queue,+ **({"headers": fairness_headers} if fairness_headers else {}),+ )+ # Dispatch FIRST; DEL the barrier keys only after dispatch succeeds.+ #+ # If apply_async raises (broker outage, serialisation error,+ # routing failure), the exception propagates. The barrier keys+ # are NOT yet DEL'd, so:+ # - Subsequent late-arriving link tasks (if any) hit the Lua+ # ``< 0`` branch (counter is still 0 — DECR-on-zero would+ # yield -1) and exit cleanly via the abandoned branch.+ # - The keys TTL-expire eventually — Redis cleans them up.+ # - The link task's failure surfaces via Celery's standard+ # task-failure channels.+ #+ # If we DEL'd before apply_async (the previous design), an+ # apply_async failure would strand the execution with no keys,+ # no TTL, no Celery retry (max_retries=0), and no link_error+ # fallback — silent infinite hang. The deferred DEL is what+ # restores parity-or-better with the chord baseline.+ callback_result = callback_signature.apply_async()+ redis_client.delete(+ _remaining_key(execution_id),+ _results_key(execution_id),+ )++ logger.info(+ f"[exec:{execution_id}] Barrier complete — "+ f"fired callback {callback_task_name} on {callback_queue} "+ f"with {len(all_results)} aggregated results "+ f"(callback_task_id={callback_result.id})"+ )++ return {+ "status": "complete",+ "callback_task_id": callback_result.id,+ "aggregated_count": len(all_results),+ }+++@worker_task(+ name="barrier_abort",+ # Same posture as ``barrier_decr_and_check`` — non-idempotent at+ # the Redis level (DEL is fine to repeat, but we don't want+ # double-logged abort events polluting Sentry).+ max_retries=0,+)+def barrier_abort(+ request: Any = None,+ exc: Any = None,+ traceback: Any = None,+ *,+ execution_id: str,+) -> dict[str, Any]:+ """``link_error`` callback: header task failed → clean up barrier state.++ Celery 5.5 invokes new-style errbacks as+ ``errback(request, exc, traceback)`` only for ``bind=False`` tasks+ with arity > 1. If a worker hits the ``NotRegistered`` fallback+ during a mixed-version rolling deploy (or if this task is ever+ switched to ``bind=True``), Celery calls the errback old-style+ with just ``(task_id,)``. The defaults on ``request`` / ``exc`` /+ ``traceback`` mean the old-style path degrades to a clean+ ``exc=None`` log line rather than a confusing+ ``TypeError: missing required positional arguments``.++ Mirrors Celery chord's default error semantic: when any header+ task fails, the chord callback isn't invoked. Without explicit+ cleanup the ``remaining`` counter would stay above 0 until TTL+ expiry, the ``results`` key would leak, and any subsequent+ successful header tasks' link would still DECR — eventually+ spurious-firing the callback with partial results (or via the+ Lua ``< 0`` branch, abandoning the execution but only after the+ extra task work).++ We DEL the barrier keys here so the in-flight successful tasks'+ link DECR returns negative → Lua ``< 0`` branch fires →+ aggregation cleanly abandoned. **Terminal status drivers**: this+ task does NOT mark the workflow FAILED — the outer orchestrators+ (``_run_workflow_api`` / ``_orchestrate_file_processing_general``)+ wrap only the *synchronous* fan-out in try/except and have+ already returned by the time ``barrier_abort`` runs. Workflow+ terminal status on async header failure is driven by per-file+ updates inside ``process_file_batch`` plus the eventual+ aggregating callback (which won't fire because we DEL'd the+ keys). A future Phase 6b' refinement may have ``barrier_abort``+ itself drive the execution-level status update.++ **Concurrent-failure dedup.** Every header task attaches the same+ ``link_error``, so N simultaneous failures would otherwise fire N+ ``barrier_abort`` executions — each calling ``logger.error`` and+ each producing its own Sentry event for what is a *single* logical+ execution failure. A ``SET NX`` lock on+ ``barrier:abort_lock:{execution_id}`` makes the first abort win;+ subsequent aborts for the same execution see the lock present and+ early-exit silently. The DELs are idempotent anyway (DEL on a+ missing key is a no-op), but the dedup collapses the alert noise.++ **Lock release on DELETE failure.** If SET NX succeeds (this+ abort wins) but the subsequent DELETE then raises (mid-abort+ Redis blip), the lock stays held while the barrier keys are NOT+ deleted. Every sibling ``barrier_abort`` would then hit the+ dedup early-exit and do nothing, leaving the counter alive for+ in-flight successful tasks to DECR to 0 → callback fires with+ partial results → silent success masking. To close this hole,+ we explicitly release the abort_lock if the DELETE raises, so+ a sibling abort can re-acquire it and complete the cleanup.+ """+ redis_client = _get_redis_client()+ # ``SET NX EX`` — first-write-wins with auto-expiry. TTL matches+ # the barrier keys' TTL so the lock cleans up alongside them.+ # The lock is also explicitly DEL'd by ``enqueue`` at the start+ # of any new execution with the same execution_id, to prevent a+ # stale lock from masking a retry's failure as success.+ if not redis_client.set(+ _abort_lock_key(execution_id), "1", ex=_key_ttl_seconds(), nx=True+ ):+ # Another barrier_abort for this execution already won the+ # SET NX race. Silently exit — no log, no Sentry, no+ # duplicate DEL.+ return {+ "status": "deduplicated",+ "execution_id": execution_id,+ }+ try:+ redis_client.delete(+ _remaining_key(execution_id),+ _results_key(execution_id),+ )+ except Exception:+ # DELETE failed AFTER we won the SET NX. Release the lock so+ # subsequent aborts aren't dedup-suppressed into no-ops while+ # the barrier keys still exist. ``suppress`` because if the+ # lock-release also fails, the original DELETE exception is+ # the more useful one to propagate (Redis is clearly in+ # trouble and the original error captures that).+ with contextlib.suppress(Exception):+ redis_client.delete(_abort_lock_key(execution_id))+ raise+ logger.error(+ f"[exec:{execution_id}] Header task failed; barrier aborted. "+ f"Cleaned up remaining/results keys. exc={exc!r}"+ )+ return {+ "status": "aborted",+ "execution_id": execution_id,+ "reason": str(exc),+ }diff --git a/workers/sample.env b/workers/sample.envindex 5c0bce2d4..4764fb0c5 100644--- a/workers/sample.env+++ b/workers/sample.env@@ -92,6 +92,45 @@ CACHE_REDIS_SSL_CERT_REQS=required# When True: point CACHE_REDIS_HOST to Sentinel K8s service DNS, CACHE_REDIS_PORT to 26379CACHE_REDIS_SENTINEL_MODE=False+# =============================================================================+# Distributed Barrier+# =============================================================================+# Selects the substrate that coordinates chord-style fan-out + callback:+# chord (default) — celery.chord(header)(body); current production behaviour+# redis — Redis DECR remaining:{exec_id} + RPUSH results+# aggregation. Replaces only the chord aggregation+# primitive; dispatch / retries / monitoring still ride+# on Celery. Avoids the documented silent-drop class+# in Celery's chord backend at large scale.+# Default chord keeps existing behaviour exactly — unset / missing env+# is safe.+WORKER_BARRIER_BACKEND=chord++# Optional separate Redis connection for the barrier counter ++# results-aggregation keys. Falls back to REDIS_* (the canonical worker+# Redis instance) if unset. Set these only if you want the barrier on a+# different Redis from the cache layer.+# WORKER_BARRIER_REDIS_HOST=unstract-redis+# WORKER_BARRIER_REDIS_PORT=6379+# WORKER_BARRIER_REDIS_DB=0+# WORKER_BARRIER_REDIS_PASSWORD=+# WORKER_BARRIER_REDIS_SENTINEL_MODE=False++# Belt-and-suspenders cleanup TTL for orphaned barrier counter ++# results keys. On the happy path the keys are explicitly DEL'd when+# the last task completes — TTL only matters when the counter stalls+# (e.g. some header tasks fail and the link never fires for them).+#+# MUST be strictly > longest plausible execution wall-clock. If TTL+# expires while tasks are still in flight, the next successful task+# fires a spurious callback with just its own result.+#+# 6h default gives 2-3x margin over FILE_PROCESSING_TASK_TIME_LIMIT+# (this sample ships 2h; docker/sample.env up to 3h; process_file_batch+# has max_retries=0 so no retry amplification). Raise this if your+# workflows can plausibly span >6h.+WORKER_BARRIER_KEY_TTL_SECONDS=21600+# Database URL (for fallback usage)DATABASE_URL=postgresql://unstract_dev:unstract_pass@unstract-db:5432/unstract_dbdiff --git a/workers/shared/workflow/execution/orchestration_utils.py b/workers/shared/workflow/execution/orchestration_utils.pyindex 6d4cf7ff4..5b7167774 100644--- a/workers/shared/workflow/execution/orchestration_utils.py+++ b/workers/shared/workflow/execution/orchestration_utils.py@@ -4,22 +4,29 @@ This module provides standardized workflow orchestration patterns,chord execution, batch processing, and task coordination utilities."""+from __future__ import annotations+import os-from typing import Any+from typing import TYPE_CHECKING, Any-from queue_backend import CeleryChordBarrier, FairnessKey+from queue_backend import BarrierHandle, FairnessKey, get_barrierfrom ...enums import FileDestinationType, PipelineTypefrom ...enums.worker_enums import QueueNamefrom ...infrastructure.logging import WorkerLogger+if TYPE_CHECKING:+ from celery.canvas import Signature+logger = WorkerLogger.get_logger(__name__)# Single ``Barrier`` instance reused across all-# ``WorkflowOrchestrationUtils.create_chord_execution`` calls. A future-# Phase 6b will replace this with a factory call (``get_barrier()``)-# that picks the impl from ``WORKER_BARRIER_BACKEND``.-_BARRIER = CeleryChordBarrier()+# ``WorkflowOrchestrationUtils.create_chord_execution`` calls. The+# substrate is selected at module-import (worker startup) time by the+# ``WORKER_BARRIER_BACKEND`` env var (default ``chord``). Flag flips+# require a pod restart — same posture as every other ``WORKER_*``+# env in the codebase.+_BARRIER = get_barrier()class WorkflowOrchestrationUtils:@@ -27,14 +34,14 @@ class WorkflowOrchestrationUtils:@staticmethoddef create_chord_execution(- batch_tasks: list[Any],+ batch_tasks: list[Signature],callback_task_name: str,callback_kwargs: dict[str, Any],callback_queue: str,app_instance: Any,*,fairness: FairnessKey | None = None,- ) -> Any:+ ) -> BarrierHandle | None:"""Standardized fan-out + callback pattern (Phase 6 ``Barrier``).Routes through ``CeleryChordBarrier`` — a thin wrapper arounddiff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.pyindex 8d24658ca..e61bc4a18 100644--- a/workers/tests/test_barrier.py+++ b/workers/tests/test_barrier.py@@ -28,7 +28,6 @@ The single ``chord(...)`` call still lives infrom __future__ import annotations-import contextlibfrom dataclasses import dataclassfrom unittest.mock import MagicMock, patch@@ -105,12 +104,13 @@ class TestBarrierProtocolShape:## ``required_attrs`` is hand-maintained: if a future refactor# adds a required attribute to ``TaskHandle`` /- # ``BarrierHandle``, add it here too. We do *not* introspect- # ``TaskHandle.__annotations__`` because runtime-checkable- # Protocols would force every substrate handle (including- # third-party ones we don't control) to be runtime-checkable- # too. Trade-off: the maintenance is manual but the contract- # is enforced.+ # ``BarrierHandle``, add it here too. The hand-maintained form+ # is grepable next to the comment that explains it, and+ # ``TaskHandle`` is intentionally minimal (``id: str``) so the+ # manual-update burden is one line every several phases.+ # ``__annotations__`` introspection would also work (and+ # doesn't require ``@runtime_checkable``), but the explicit+ # tuple keeps the contract surface explicit.async_result_instance = AsyncResult("placeholder-task-id")required_attrs = ("id",)missing = [@@ -557,10 +557,8 @@ def _setup_workflow_api_mocks(mock_create_chord = MagicMock(name="create_chord_execution")# Truthy handle on the success path; ``None`` on the two failure- # paths. Production's ``if not result:`` branch (today; a future- # phase-6b PR may tighten to ``if result is None:`` — see the- # TODO at the call site in ``api-deployment/tasks.py``) then- # distinguishes the two ``None`` cases via ``batch_tasks`` length.+ # paths. Production's ``if result is None:`` branch then+ # distinguishes the two cases via ``batch_tasks`` length.if chord_outcome == "success":mock_create_chord.return_value = MagicMock(id="chord-result-id")else:@@ -720,12 +718,13 @@ class TestCallSiteFairnessContracts:mock_create_chord,)- # ``contextlib.suppress`` over a bare ``except Exception: pass``- # to declare intent explicitly: any post-``create_chord_execution``- # raise (e.g. the mocked api_client's status update path) is- # tolerated, because the next ``assert mock_create_chord.called``- # is what proves the chord call was actually reached.- with contextlib.suppress(Exception):+ # Tolerate post-``create_chord_execution`` raises (e.g. the+ # mocked api_client's status-update path), but re-raise if+ # ``create_chord_execution`` was never invoked — a pre-chord+ # regression would otherwise be silently swallowed and surface+ # as a misleading ``assert mock_create_chord.called`` failure+ # rather than the actual error.+ try:general_tasks._orchestrate_file_processing_general(api_client=api_client,workflow_id="wf-1",@@ -737,6 +736,9 @@ class TestCallSiteFairnessContracts:use_file_history=False,organization_id="org_test",)+ except Exception:+ if not mock_create_chord.called:+ raiseassert mock_create_chord.called, ("_orchestrate_file_processing_general did not call "diff --git a/workers/tests/test_barrier_backend_selection.py b/workers/tests/test_barrier_backend_selection.pynew file mode 100644index 000000000..7a452652e--- /dev/null+++ b/workers/tests/test_barrier_backend_selection.py@@ -0,0 +1,116 @@+"""Tests for the ``get_barrier()`` factory + ``WORKER_BARRIER_BACKEND`` flag.++Phase 6b selects the ``Barrier`` substrate at runtime via an env flag.+This test suite pins:++- Default (env unset) → ``CeleryChordBarrier`` — protects against any+ production behaviour change at merge time; downstream envs that+ haven't added the new flag still run the same substrate they ran+ yesterday.+- ``WORKER_BARRIER_BACKEND=chord`` → ``CeleryChordBarrier``.+- ``WORKER_BARRIER_BACKEND=redis`` → ``RedisDecrBarrier``.+- Unknown values raise ``ValueError`` — silent fallback to default+ would mask a typo'd flag in production env, which the canary+ must not allow.+- Case + whitespace tolerance — operators frequently set env vars+ via shell scripts or k8s configmaps; ``" REDIS "`` should+ resolve the same as ``"redis"``.+- ``BarrierBackend`` enum membership — the canonical string values+ match what operators set in env.+"""++from __future__ import annotations++import pytest++from queue_backend import (+ BarrierBackend,+ CeleryChordBarrier,+ RedisDecrBarrier,+ get_barrier,+)+++class TestBarrierBackendEnum:+ def test_chord_value_is_canonical_string(self):+ """The enum value is what operators set in env. If this+ drifts, the factory's dispatch breaks silently for one+ substrate (the one whose enum value no longer matches the+ documented env value)."""+ assert BarrierBackend.CHORD.value == "chord"+ assert BarrierBackend.REDIS.value == "redis"++ def test_str_enum_string_equality(self):+ """``StrEnum`` members compare equal to their string values —+ means tests + production code can use either form+ interchangeably without conversion."""+ assert BarrierBackend.CHORD == "chord"+ assert BarrierBackend.REDIS == "redis"+++class TestGetBarrierFactory:+ def test_default_returns_celery_chord_barrier(self, monkeypatch):+ """Env unset → ``CeleryChordBarrier``. This is what protects+ production from a behaviour change at merge time — downstream+ envs (``unstract-cloud``, self-hosted, etc.) that haven't+ added the new flag run the same substrate they ran yesterday."""+ monkeypatch.delenv("WORKER_BARRIER_BACKEND", raising=False)+ barrier = get_barrier()+ assert isinstance(barrier, CeleryChordBarrier)++ def test_chord_selects_celery_chord_barrier(self, monkeypatch):+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.CHORD)+ assert isinstance(get_barrier(), CeleryChordBarrier)++ def test_redis_selects_redis_decr_barrier(self, monkeypatch):+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.REDIS)+ assert isinstance(get_barrier(), RedisDecrBarrier)++ def test_unknown_value_raises_loudly(self, monkeypatch):+ """Silent fallback to default would mask a typo'd production+ env (``rediz``, ``REDDIS``, ``redis-decr``) — the canary+ must surface the misconfiguration."""+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", "rediz")+ with pytest.raises(ValueError, match=r"rediz"):+ get_barrier()++ def test_empty_string_raises_loudly(self, monkeypatch):+ """Empty string is not the same as "unset" — it's a positive+ misconfiguration (e.g. an unsubstituted ``$BARRIER`` in a+ shell script). Surface it rather than falling back to+ default, which would mask the bug."""+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", "")+ with pytest.raises(ValueError):+ get_barrier()++ def test_case_insensitive(self, monkeypatch):+ """Operators frequently set env vars via shell heredocs or+ k8s configmaps with inconsistent casing. ``REDIS`` and+ ``Redis`` should resolve the same as ``redis``."""+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", "REDIS")+ assert isinstance(get_barrier(), RedisDecrBarrier)+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", "Chord")+ assert isinstance(get_barrier(), CeleryChordBarrier)++ def test_whitespace_tolerated(self, monkeypatch):+ """Stray whitespace (common from shell substitutions) is+ stripped before the membership check."""+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", " redis ")+ assert isinstance(get_barrier(), RedisDecrBarrier)++ def test_returns_fresh_instance_per_call(self, monkeypatch):+ """``get_barrier()`` builds a new instance each call rather+ than returning a singleton. Call sites that want a singleton+ (e.g. ``orchestration_utils._BARRIER``) cache it at module+ load time. Tests can flip the env per-test without leaking+ a stale instance from a previous test."""+ monkeypatch.setenv("WORKER_BARRIER_BACKEND", "chord")+ a = get_barrier()+ b = get_barrier()+ assert a is not b+ assert isinstance(a, CeleryChordBarrier)+ assert isinstance(b, CeleryChordBarrier)+++if __name__ == "__main__":+ pytest.main([__file__, "-v"])diff --git a/workers/tests/test_barrier_differential.py b/workers/tests/test_barrier_differential.pynew file mode 100644index 000000000..99bbb1200--- /dev/null+++ b/workers/tests/test_barrier_differential.py@@ -0,0 +1,232 @@+"""Differential test — both ``Barrier`` substrates produce equivalent+callback firings for a fixed input.++This is the load-bearing test that justifies the substrate swap:+given the same header tasks + callback descriptor + fairness, both+``CeleryChordBarrier`` and ``RedisDecrBarrier`` must ultimately fire+the same callback with the same ``(task_name, args, kwargs, queue,+headers)``. The execution paths diverge — chord aggregates via+Celery's chord backend, redis aggregates via ``DECR remaining`` ++``RPUSH`` — but the observable output (what the callback receives)+is equivalent.++If this test fails, the substrate swap is *not* safe to roll out:+the callback would see different aggregated results or different+metadata between substrates, breaking executions when the flag+flips.++Note this is a unit-level differential — it pins the wire shape of+the callback dispatch, not full end-to-end equivalence. A future+integration test (against real Celery + Redis with N parallel+workers) would belong in ``test_barrier_integration.py``.+"""++from __future__ import annotations++import json+from unittest.mock import MagicMock, patch++import pytest++from queue_backend import CeleryChordBarrier, RedisDecrBarrier, barrier_decr_and_check+from queue_backend.fairness import FairnessKey, WorkloadType+++# A fixed scenario both substrates run through.+_HEADER_TASK_RESULTS = [+ {"batch_id": "b1", "files": ["f1", "f2"], "status": "ok"},+ {"batch_id": "b2", "files": ["f3", "f4"], "status": "ok"},+ {"batch_id": "b3", "files": ["f5"], "status": "ok"},+]+_CALLBACK_TASK = "process_batch_callback_api"+_CALLBACK_KWARGS = {+ "execution_id": "exec-differential-1",+ "pipeline_id": "pipe-x",+ "organization_id": "org-y",+}+_CALLBACK_QUEUE = "api_file_processing_callback"+_FAIRNESS = FairnessKey(org_id="org-y", workload_type=WorkloadType.API)+++def _capture_chord_callback_dispatch() -> dict:+ """Run ``CeleryChordBarrier.enqueue`` against a mocked ``chord``+ and return the callback signature's construction args.++ Chord's wire model: ``chord(header_tasks)(callback_signature)`` —+ the callback_signature is built via ``app.signature(name, kwargs=...,+ queue=..., headers=...)`` and then passed as the body.+ """+ app = MagicMock(name="app")+ headers_captured = {}++ def signature_recorder(name, **kwargs):+ headers_captured["task_name"] = name+ headers_captured["args"] = kwargs.get("args") # None for chord callback+ headers_captured["kwargs"] = kwargs.get("kwargs")+ headers_captured["queue"] = kwargs.get("queue")+ headers_captured["headers"] = kwargs.get("headers")+ return MagicMock(name="callback_signature")++ app.signature.side_effect = signature_recorder+ header = [MagicMock(name=f"h{i}") for i in range(len(_HEADER_TASK_RESULTS))]++ with patch("queue_backend.barrier.chord"):+ CeleryChordBarrier().enqueue(+ header,+ callback_task_name=_CALLBACK_TASK,+ callback_kwargs=_CALLBACK_KWARGS,+ callback_queue=_CALLBACK_QUEUE,+ app_instance=app,+ fairness=_FAIRNESS,+ )+ return headers_captured+++def _capture_redis_callback_dispatch() -> dict:+ """Drive ``RedisDecrBarrier.enqueue`` + simulate the link task+ completing for all header tasks, then capture the final+ callback signature's construction args at the post-aggregation+ dispatch site.+ """+ redis_mock = MagicMock(name="redis")+ barrier = RedisDecrBarrier(redis_client=redis_mock)+ header = [MagicMock(name=f"h{i}") for i in range(len(_HEADER_TASK_RESULTS))]+ barrier.enqueue(+ header,+ callback_task_name=_CALLBACK_TASK,+ callback_kwargs=_CALLBACK_KWARGS,+ callback_queue=_CALLBACK_QUEUE,+ app_instance=MagicMock(name="app"),+ fairness=_FAIRNESS,+ )++ # Now simulate each link firing. The first N-1 see remaining > 0;+ # the last sees remaining == 0 and dispatches the callback.+ headers_captured = {}++ def signature_recorder(name, **kwargs):+ headers_captured["task_name"] = name+ headers_captured["args"] = kwargs.get("args")+ headers_captured["kwargs"] = kwargs.get("kwargs")+ headers_captured["queue"] = kwargs.get("queue")+ headers_captured["headers"] = kwargs.get("headers")+ cb_sig = MagicMock(name="callback_signature")+ cb_sig.apply_async.return_value = MagicMock(id="callback-task-id")+ return cb_sig++ # Simulate the Lua script's return for each link invocation.+ # First N-1 calls: [remaining, []]. Last call: [0, [serialised results]].+ n = len(_HEADER_TASK_RESULTS)+ serialised_results = [json.dumps(r) for r in _HEADER_TASK_RESULTS]+ return_values = [+ [n - 1 - i, []] for i in range(n - 1)+ ] + [[0, serialised_results]]+ script_mock = MagicMock(side_effect=return_values)++ with patch("queue_backend.redis_barrier._get_redis_client") as get_redis:+ # Inside the link task, ``_get_redis_client()`` is called to+ # build the script invocation. Return a mock whose+ # register_script returns the script_mock that yields the+ # sequenced return values.+ link_redis = MagicMock(name="link_redis")+ link_redis.register_script.return_value = script_mock+ get_redis.return_value = link_redis++ with patch("celery.current_app") as current_app:+ current_app.signature.side_effect = signature_recorder+ for header_result in _HEADER_TASK_RESULTS:+ barrier_decr_and_check.run(+ header_result,+ execution_id=_CALLBACK_KWARGS["execution_id"],+ callback_descriptor={+ "task_name": _CALLBACK_TASK,+ "kwargs": _CALLBACK_KWARGS,+ "queue": _CALLBACK_QUEUE,+ "fairness_headers": _FAIRNESS.as_header(),+ },+ )+ return headers_captured+++class TestBarrierSubstrateDifferential:+ """Both substrates must produce equivalent callback firings.++ A failure here means the swap is not safe — flag flipping in+ Phase 6c would change the data the callback receives.+ """++ def test_same_callback_task_name(self):+ """Both substrates fire the same callback task name."""+ chord_cap = _capture_chord_callback_dispatch()+ redis_cap = _capture_redis_callback_dispatch()+ assert chord_cap["task_name"] == redis_cap["task_name"] == _CALLBACK_TASK++ def test_same_callback_kwargs(self):+ """Both substrates forward the same ``execution_id`` /+ ``pipeline_id`` / ``organization_id`` to the callback."""+ chord_cap = _capture_chord_callback_dispatch()+ redis_cap = _capture_redis_callback_dispatch()+ assert chord_cap["kwargs"] == redis_cap["kwargs"] == _CALLBACK_KWARGS++ def test_same_callback_queue(self):+ """Both substrates route the callback to the same queue —+ without this, post-Phase-6c the callback could land on a+ worker that doesn't have it registered."""+ chord_cap = _capture_chord_callback_dispatch()+ redis_cap = _capture_redis_callback_dispatch()+ assert chord_cap["queue"] == redis_cap["queue"] == _CALLBACK_QUEUE++ def test_same_fairness_headers(self):+ """Both substrates ride the same ``x-fairness-key`` AMQP+ header on the callback. Mismatch would mean post-flag-flip+ executions route through different fairness slots, which+ the L1/L2/L3 admission logic could throttle differently."""+ chord_cap = _capture_chord_callback_dispatch()+ redis_cap = _capture_redis_callback_dispatch()+ assert chord_cap["headers"] == redis_cap["headers"] == _FAIRNESS.as_header()++ def test_redis_substrate_passes_aggregated_results_as_first_arg(self):+ """Chord's callback receives ``[result_1, result_2, ...]`` via+ Celery's chord-aggregator (an implicit first positional arg+ Celery injects). The Redis substrate must replicate the+ same shape — pass the aggregated list explicitly via+ ``args=[all_results]`` on the callback signature.++ We can't observe chord's implicit arg injection from the+ outside (it's the chord backend's job), but we CAN pin the+ Redis substrate's behaviour: when aggregation completes,+ the callback signature's ``args[0]`` MUST equal the+ original header-task results list. That's the contract the+ callback already depends on (``file_batch_results: list[dict]``+ — see callback/tasks.py:1552)."""+ redis_cap = _capture_redis_callback_dispatch()+ assert redis_cap["args"] == [_HEADER_TASK_RESULTS], (+ f"Redis substrate's callback args must be a list "+ f"containing one list-of-results (matching chord's "+ f"aggregator shape). Got: {redis_cap['args']!r}"+ )++ def test_both_handle_empty_headers_identically(self):+ """Empty header → ``None`` from both substrates. The caller's+ ``if result is None:`` branch then handles the zero-files+ contract uniformly."""+ chord_result = CeleryChordBarrier().enqueue(+ [],+ callback_task_name=_CALLBACK_TASK,+ callback_kwargs=_CALLBACK_KWARGS,+ callback_queue=_CALLBACK_QUEUE,+ app_instance=MagicMock(),+ )+ redis_result = RedisDecrBarrier(redis_client=MagicMock()).enqueue(+ [],+ callback_task_name=_CALLBACK_TASK,+ callback_kwargs=_CALLBACK_KWARGS,+ callback_queue=_CALLBACK_QUEUE,+ app_instance=MagicMock(),+ )+ assert chord_result is None+ assert redis_result is None+++if __name__ == "__main__":+ pytest.main([__file__, "-v"])diff --git a/workers/tests/test_queue_backend_seam.py b/workers/tests/test_queue_backend_seam.pyindex 7075dbaf2..8cd827cc1 100644--- a/workers/tests/test_queue_backend_seam.py+++ b/workers/tests/test_queue_backend_seam.py@@ -276,14 +276,21 @@ class TestPublicSurface:import queue_backend# Phase 6a added Barrier / BarrierHandle / CeleryChordBarrier.- # Phase 6b will add a ``get_barrier`` factory when the- # ``WORKER_BARRIER_BACKEND`` flag wiring lands.+ # Phase 6b adds RedisDecrBarrier + barrier_decr_and_check+ # (registered as a Celery task on import) + the BarrierBackend+ # enum + the get_barrier factory that the WORKER_BARRIER_BACKEND+ # env flag drives.assert set(queue_backend.__all__) == {"Barrier",+ "BarrierBackend","BarrierHandle","CeleryChordBarrier","FairnessKey",+ "RedisDecrBarrier",+ "barrier_abort",+ "barrier_decr_and_check","dispatch",+ "get_barrier","worker_task",}diff --git a/workers/tests/test_redis_barrier.py b/workers/tests/test_redis_barrier.pynew file mode 100644index 000000000..de098cf58--- /dev/null+++ b/workers/tests/test_redis_barrier.py@@ -0,0 +1,972 @@+"""Characterisation tests for ``RedisDecrBarrier`` (PG Queue Phase 6b).++The risky behaviour-change PR replacing Celery's chord aggregation+primitive with the labs-design ``DECR remaining`` + ``RPUSH results``+pattern (see ``queue_backend/redis_barrier.py``).++Three layers:++1. **Protocol shape** — ``RedisDecrBarrier`` satisfies the ``Barrier``+ Protocol; the handle satisfies ``BarrierHandle``.+2. **Wire model** — ``enqueue`` initialises the per-execution counter+ + results list with TTL, stamps fairness on each header task,+ attaches the ``barrier_decr_and_check`` link, and dispatches via+ ``apply_async``.+3. **Link aggregation** — the ``barrier_decr_and_check`` task runs the+ atomic Lua script, fires the callback exactly when remaining reads+ 0, passes the aggregated results, and respects fairness on the+ callback signature.++Redis is mocked via ``MagicMock`` — the contract under test is the+sequence of Redis ops + their args, not Redis's own correctness.+A future integration test (against a real Redis) would belong in+``test_redis_barrier_integration.py``.+"""++from __future__ import annotations++import json+from unittest.mock import MagicMock, patch++import pytest++from queue_backend import (+ Barrier,+ BarrierHandle,+ RedisDecrBarrier,+ barrier_decr_and_check,+)+from queue_backend.fairness import FAIRNESS_HEADER_NAME, FairnessKey, WorkloadType+from queue_backend.redis_barrier import (+ _KEY_TTL_DEFAULT_SECONDS,+ _remaining_key,+ _results_key,+)+++# --- Protocol shape ---+++class TestRedisDecrBarrierProtocolShape:+ def test_satisfies_barrier_protocol(self):+ barrier: Barrier = RedisDecrBarrier()+ assert callable(getattr(barrier, "enqueue", None))++ def test_handle_satisfies_barrier_handle(self):+ from queue_backend.redis_barrier import _RedisBarrierHandle++ handle: BarrierHandle = _RedisBarrierHandle(id="exec-1")+ assert handle.id == "exec-1"+ assert isinstance(handle.id, str)+++# --- Enqueue: wire model ---+++def redis_mock():+ """Mock the Redis client used by ``RedisDecrBarrier``."""+ return MagicMock(name="redis_client")+++def barrier(redis_mock):+ """``RedisDecrBarrier`` with an injected mock client.++ Production builds its client from env via ``create_redis_client``;+ tests inject so no Redis is required.+ """+ return RedisDecrBarrier(redis_client=redis_mock)+++class TestRedisDecrBarrierEnqueue:+ def test_empty_header_returns_none_and_touches_no_redis(self, barrier, redis_mock):+ """Mirrors the ``CeleryChordBarrier`` zero-header contract:+ ``None`` is the *sole* signal of no-work-enqueued, and we+ don't pay the Redis round-trip for it."""+ result = barrier.enqueue(+ [],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ )+ assert result is None+ # No SET / DELETE on the empty path — short-circuit before any+ # Redis round-trip.+ redis_mock.set.assert_not_called()+ redis_mock.delete.assert_not_called()++ def test_missing_execution_id_raises(self, barrier):+ """``RedisDecrBarrier`` uses ``execution_id`` as the key suffix+ for ``remaining:{exec_id}`` / ``results:{exec_id}``. Without+ it we can't isolate this execution's counter — fail loudly+ instead of routing into a global namespace collision."""+ header = [MagicMock(name="h1")]+ with pytest.raises(ValueError, match=r"execution_id"):+ barrier.enqueue(+ header,+ callback_task_name="cb",+ # No execution_id.+ callback_kwargs={"pipeline_id": "p-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ )++ def test_initialises_remaining_counter_with_ttl(self, barrier, redis_mock):+ """``remaining:{exec_id}`` SET to ``len(header_tasks)`` with+ TTL — belt-and-suspenders cleanup if a link never fires for+ a task (e.g. worker crash mid-execution)."""+ header = [MagicMock(name="h1"), MagicMock(name="h2"), MagicMock(name="h3")]+ barrier.enqueue(+ header,+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-42"},+ callback_queue="q",+ app_instance=MagicMock(),+ )+ redis_mock.set.assert_called_once_with(+ _remaining_key("exec-42"),+ 3,+ ex=_KEY_TTL_DEFAULT_SECONDS,+ )+ # ``results:{exec_id}`` AND ``abort_lock:{exec_id}`` are DEL'd+ # to clear any stale prior-run leftovers (e.g. from a retry+ # reusing the same execution_id). Without DELing the abort+ # lock here, a stale lock from a prior failed run would mask+ # a retry's failure as success.+ from queue_backend.redis_barrier import _abort_lock_key++ redis_mock.delete.assert_called_once_with(+ _results_key("exec-42"),+ _abort_lock_key("exec-42"),+ )++ def test_link_attached_to_each_header_task(self, barrier, redis_mock):+ """Every header task gets the ``barrier_decr_and_check`` link.+ Cloning (not mutating the caller's list) preserves the same+ guarantee ``CeleryChordBarrier`` makes for cross-tenant+ signature reuse."""+ h1 = MagicMock(name="h1")+ h2 = MagicMock(name="h2")+ barrier.enqueue(+ [h1, h2],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ )+ # Each task cloned, .link(...) called once on the clone,+ # apply_async called once on the clone.+ h1.clone.assert_called_once_with()+ h2.clone.assert_called_once_with()+ h1.clone.return_value.link.assert_called_once()+ h2.clone.return_value.link.assert_called_once()+ h1.clone.return_value.apply_async.assert_called_once_with()+ h2.clone.return_value.apply_async.assert_called_once_with()+ # The caller's signatures were NOT mutated (no .link on the+ # originals).+ h1.link.assert_not_called()+ h2.link.assert_not_called()++ def test_fairness_header_stamped_on_each_header_task(self, barrier, redis_mock):+ """``fairness.as_header()`` rides on every header signature as+ an additive AMQP header — same wire shape as+ ``CeleryChordBarrier``'s fairness plumbing."""+ h1 = MagicMock(name="h1")+ fairness = FairnessKey(org_id="org-x", workload_type=WorkloadType.API)+ barrier.enqueue(+ [h1],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ fairness=fairness,+ )+ # ``as_header()`` returns ``{FAIRNESS_HEADER_NAME: <dict>}``+ # (a nested dict, not a JSON string — see fairness.py:62).+ h1.clone.return_value.set.assert_called_once_with(+ headers=fairness.as_header()+ )++ def test_no_fairness_no_header_added(self, barrier, redis_mock):+ """When ``fairness=None``, no ``headers=`` kwarg on the+ signature — preserves wire equivalence for the+ non-fairness call path."""+ h1 = MagicMock(name="h1")+ barrier.enqueue(+ [h1],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ )+ h1.clone.return_value.set.assert_not_called()++ def test_returns_handle_with_execution_id(self, barrier, redis_mock):+ """``BarrierHandle.id`` is the execution id — what call sites+ log for chord-id tracing. (Under ``CeleryChordBarrier`` this+ was the chord aggregator's AsyncResult id; under+ ``RedisDecrBarrier`` we don't have such a task, so we expose+ the execution id which is a stable per-execution identifier+ that the existing log consumers don't depend on the+ underlying type of.)"""+ result = barrier.enqueue(+ [MagicMock()],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-42"},+ callback_queue="q",+ app_instance=MagicMock(),+ )+ assert result is not None+ assert result.id == "exec-42"+ assert isinstance(result.id, str)++ def test_setup_failure_raises(self, barrier, redis_mock):+ """Any substrate failure (Redis down, network blip mid-setup)+ propagates as an exception — matches the ``Barrier`` Protocol:+ ``None`` is sole "no-op" signal, everything else raises."""+ redis_mock.set.side_effect = ConnectionError("redis is down")+ with pytest.raises(ConnectionError, match="redis is down"):+ barrier.enqueue(+ [MagicMock()],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ )++ def test_default_ttl_is_six_hours(self):+ """Lock in the chosen default. Accidental shrinkage to a few+ minutes would cause spurious callback fires on any execution+ longer than the new value; accidental expansion to days would+ leak orphaned keys. The default reflects the worst-case+ ``FILE_PROCESSING_TASK_TIME_LIMIT`` (3h) x 2x margin."""+ assert _KEY_TTL_DEFAULT_SECONDS == 6 * 60 * 60++ def test_ttl_overridable_via_env(self, barrier, redis_mock, monkeypatch):+ """Operators with substantially longer (or shorter) workflows+ can override the cleanup TTL via ``WORKER_BARRIER_KEY_TTL_SECONDS``.++ Set well above the default to confirm the env-driven path is+ actually read (not the constant)."""+ monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", str(3 * 24 * 3600))+ barrier.enqueue(+ [MagicMock()],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ )+ # ``ex=`` on the counter SET should be the overridden 3-day value.+ assert redis_mock.set.call_args.kwargs.get("ex") == 3 * 24 * 3600++ # Invalid-value handling tests live in ``TestTtlEnvValidation``+ # below — they now assert ``raises`` (matching ``get_barrier()``'s+ # loud-on-misconfig posture) rather than silent fallback.+++# --- Link task: aggregation ---+++class TestBarrierDecrAndCheckLink:+ """The ``barrier_decr_and_check`` worker task — the load-bearing+ aggregator. Runs after each header task and either decrements ++ aggregates (most calls) or fires the callback (the last call).+ """++ @patch("queue_backend.redis_barrier._get_redis_client")+ def test_pending_path_decrement_only(self, get_redis):+ """When the post-decrement counter reads >0, no callback+ dispatch — just ``RPUSH + DECR`` and return a pending status."""+ redis_client = MagicMock(name="redis")+ # Lua script returns [remaining, results-empty-list-since-pending].+ # ``register_script`` returns a callable that returns the script result.+ script = MagicMock(name="script", return_value=[2, []])+ redis_client.register_script.return_value = script+ get_redis.return_value = redis_client++ with patch("celery.current_app") as current_app:+ # Invoke the underlying function directly (bypass+ # Celery's @worker_task wrapper for unit testing).+ result = barrier_decr_and_check.run(+ {"batch_id": "b1", "status": "ok"},+ execution_id="exec-1",+ callback_descriptor={+ "task_name": "cb",+ "kwargs": {"execution_id": "exec-1"},+ "queue": "q",+ "fairness_headers": None,+ },+ )++ # Lua script invoked with the right keys + JSON-serialised+ # result + TTL seconds for the EXPIRE inside the script.+ script.assert_called_once()+ call_kwargs = script.call_args.kwargs+ assert call_kwargs["keys"] == [+ _remaining_key("exec-1"),+ _results_key("exec-1"),+ ]+ assert json.loads(call_kwargs["args"][0]) == {"batch_id": "b1", "status": "ok"}+ assert call_kwargs["args"][1] == _KEY_TTL_DEFAULT_SECONDS++ # No callback dispatch — remaining > 0.+ current_app.signature.assert_not_called()+ assert result == {"status": "pending", "remaining": 2}++ @patch("queue_backend.redis_barrier._get_redis_client")+ def test_complete_path_fires_callback_with_aggregated_results(self, get_redis):+ """When the post-decrement counter reads 0, the link reads the+ full results list (returned by the Lua script in the same+ atomic step), constructs the callback signature with the+ aggregated list as the first arg, and dispatches via+ ``apply_async``."""+ redis_client = MagicMock(name="redis")+ # Lua returns [0, [serialised result_1, result_2, result_3]].+ aggregated_raw = [+ json.dumps({"batch_id": "b1", "status": "ok"}),+ json.dumps({"batch_id": "b2", "status": "ok"}),+ json.dumps({"batch_id": "b3", "status": "ok"}),+ ]+ script = MagicMock(name="script", return_value=[0, aggregated_raw])+ redis_client.register_script.return_value = script+ get_redis.return_value = redis_client++ with patch("celery.current_app") as current_app:+ cb_sig = MagicMock(name="callback_signature")+ cb_sig.apply_async.return_value = MagicMock(id="callback-task-id-xyz")+ current_app.signature.return_value = cb_sig++ result = barrier_decr_and_check.run(+ {"batch_id": "b3", "status": "ok"},+ execution_id="exec-42",+ callback_descriptor={+ "task_name": "process_batch_callback_api",+ "kwargs": {+ "execution_id": "exec-42",+ "pipeline_id": "pipe-7",+ "organization_id": "org-x",+ },+ "queue": "api_file_processing_callback",+ "fairness_headers": None,+ },+ )++ # Callback signature constructed with aggregated results as+ # first arg, callback kwargs forwarded, queue pinned.+ current_app.signature.assert_called_once_with(+ "process_batch_callback_api",+ args=[+ [+ {"batch_id": "b1", "status": "ok"},+ {"batch_id": "b2", "status": "ok"},+ {"batch_id": "b3", "status": "ok"},+ ],+ ],+ kwargs={+ "execution_id": "exec-42",+ "pipeline_id": "pipe-7",+ "organization_id": "org-x",+ },+ queue="api_file_processing_callback",+ )+ cb_sig.apply_async.assert_called_once_with()+ # DELETE is now called from Python AFTER apply_async succeeds+ # (the Lua script no longer DELs in the ``== 0`` branch). This+ # is the load-bearing deferred-cleanup invariant: a dispatch+ # failure leaves the keys + TTL in place so the execution+ # isn't stranded.+ redis_client.delete.assert_called_once_with(+ _remaining_key("exec-42"),+ _results_key("exec-42"),+ )++ assert result["status"] == "complete"+ assert result["callback_task_id"] == "callback-task-id-xyz"+ assert result["aggregated_count"] == 3++ @patch("queue_backend.redis_barrier._get_redis_client")+ def test_complete_path_passes_fairness_header_to_callback(self, get_redis):+ """Fairness rides the callback signature too — same wire shape+ as ``CeleryChordBarrier``'s callback fairness plumbing."""+ redis_client = MagicMock(name="redis")+ script = MagicMock(name="script", return_value=[0, [json.dumps({"x": 1})]])+ redis_client.register_script.return_value = script+ get_redis.return_value = redis_client++ # Match the wire shape produced by ``FairnessKey.as_header()`` —+ # a nested dict, not a JSON string (see fairness.py:62).+ fairness_headers = {+ FAIRNESS_HEADER_NAME: {+ "org_id": "org-x",+ "workload_type": "non_api",+ "pipeline_priority": 5,+ }+ }+ with patch("celery.current_app") as current_app:+ current_app.signature.return_value = MagicMock(+ apply_async=MagicMock(return_value=MagicMock(id="cb-id"))+ )++ barrier_decr_and_check.run(+ {"x": 1},+ execution_id="exec-1",+ callback_descriptor={+ "task_name": "cb",+ "kwargs": {"execution_id": "exec-1"},+ "queue": "q",+ "fairness_headers": fairness_headers,+ },+ )++ # ``headers=fairness_headers`` MUST be on the signature kwargs.+ assert current_app.signature.call_args.kwargs.get("headers") == fairness_headers++ @patch("queue_backend.redis_barrier._get_redis_client")+ def test_unserialisable_result_raises_loudly(self, get_redis):+ """A header task returning something json.dumps can't handle+ is a typed-boundary regression. Raise loudly so the+ execution surfaces an error rather than silently dropping the+ result."""+ redis_client = MagicMock(name="redis")+ get_redis.return_value = redis_client++ # ``complex`` is a classic JSON-unserialisable leaf — no+ # ``__json__`` / ``__str__``-with-meaning, and ``default=str``+ # isn't applied (we dropped it). A regressed+ # ``BatchExecutionResult.to_dict()`` producing such a leaf+ # would land here. The test pins that the loud-failure+ # ``try/except`` actually surfaces the regression rather than+ # being a no-op.+ with pytest.raises((TypeError, ValueError)):+ barrier_decr_and_check.run(+ {"bad": complex(1, 2)},+ execution_id="exec-1",+ callback_descriptor={+ "task_name": "cb",+ "kwargs": {"execution_id": "exec-1"},+ "queue": "q",+ "fairness_headers": None,+ },+ )++ def test_link_task_is_registered_with_celery_under_canonical_name(self):+ """The link task must be importable + registered under+ ``barrier_decr_and_check`` — any worker that processes the+ link task's queue needs the name to resolve.++ Mixed-version rolling deploys depend on this: a worker+ running 6b-old code without the link task registered would+ fail with ``KeyError`` on the link's task name, which is a+ loud failure (no silent drop)."""+ assert barrier_decr_and_check.name == "barrier_decr_and_check"++ def test_link_max_retries_zero_prevents_double_decrement(self):+ """The link task MUST have ``max_retries=0``. The Lua script+ is atomic at the Redis level but the link itself is not+ idempotent: a Celery retry would replay the script and+ double-RPUSH + double-DECR, corrupting the aggregation+ (counter hits 0 prematurely; results list has duplicates).+ TTL-driven cleanup is the safety net for transient failures+ rather than Celery autoretry."""+ # Celery exposes ``max_retries`` on the task class.+ assert barrier_decr_and_check.max_retries == 0++ @patch("queue_backend.redis_barrier._get_redis_client")+ def test_negative_remaining_does_not_fire_callback(self, get_redis):+ """The Lua script returns ``remaining < 0`` when the counter+ was TTL-expired or already cleaned up. The Python side must+ NOT fire the callback in that branch — otherwise every+ subsequent task completion after a TTL expiry would dispatch+ a spurious callback."""+ redis_client = MagicMock(name="redis")+ # Lua returns ``[-1, []]`` — the ``< 0`` branch where keys+ # were already DEL'd (or never existed).+ script = MagicMock(name="script", return_value=[-1, []])+ redis_client.register_script.return_value = script+ get_redis.return_value = redis_client++ with patch("celery.current_app") as current_app:+ result = barrier_decr_and_check.run(+ {"x": 1},+ execution_id="exec-1",+ callback_descriptor={+ "task_name": "cb",+ "kwargs": {"execution_id": "exec-1"},+ "queue": "q",+ "fairness_headers": None,+ },+ )++ # No callback dispatch — the abandoned branch fires.+ current_app.signature.assert_not_called()+ assert result["status"] == "abandoned"+ assert result["remaining"] == -1++ @patch("queue_backend.redis_barrier._get_redis_client")+ def test_callback_dispatch_failure_preserves_keys_for_ttl(self, get_redis):+ """If ``apply_async`` raises after the Lua ``== 0`` branch+ returned, the barrier keys MUST remain in Redis (with their+ TTL still active) so the execution isn't stranded with no+ recovery path.++ Without deferred DEL, the previous implementation DEL'd both+ keys inside the Lua script before Python's ``apply_async``+ ran. A dispatch failure (broker outage, serialisation error,+ routing failure) then left the execution with: no keys, no+ TTL, no Celery retry (``max_retries=0``), no link_error+ fallback — strictly worse than the chord baseline.++ The fix: Lua's ``== 0`` branch returns the LRANGE results+ WITHOUT DELing. Python DELs both keys only after+ ``apply_async`` succeeds. A failure raises and the TTL+ eventually cleans up the keys."""+ redis_client = MagicMock(name="redis")+ # ``== 0`` branch — script returns one result; Lua no longer DELs.+ script = MagicMock(name="script", return_value=[0, [json.dumps({"x": 1})]])+ redis_client.register_script.return_value = script+ get_redis.return_value = redis_client++ with patch("celery.current_app") as current_app:+ cb_sig = MagicMock(name="callback_signature")+ # apply_async raises — broker outage, serialisation error, etc.+ cb_sig.apply_async.side_effect = RuntimeError("broker is down")+ current_app.signature.return_value = cb_sig++ with pytest.raises(RuntimeError, match=r"broker is down"):+ barrier_decr_and_check.run(+ {"x": 1},+ execution_id="exec-1",+ callback_descriptor={+ "task_name": "cb",+ "kwargs": {"execution_id": "exec-1"},+ "queue": "q",+ "fairness_headers": None,+ },+ )++ # DELETE MUST NOT have been called — the keys must remain in+ # Redis (with their existing TTL) so:+ # - Late-arriving link tasks (if any) cleanly hit the Lua+ # ``< 0`` branch and exit via the abandoned path.+ # - TTL eventually cleans the keys.+ # - The failure is visible via Celery's standard task-failure+ # channels rather than a silent infinite hang.+ redis_client.delete.assert_not_called()+++# --- TTL env validation ---+++class TestTtlEnvValidation:+ """Misconfigured ``WORKER_BARRIER_KEY_TTL_SECONDS`` must raise.++ Matches the posture of ``get_barrier()`` on a typo'd+ ``WORKER_BARRIER_BACKEND``: a misconfigured TTL shorter than+ execution wall-clock is a correctness issue (spurious callback+ fires per the Lua ``< 0`` branch), so operators get a loud+ on-startup signal rather than silent degradation.+ """++ def test_non_integer_raises(self, barrier, redis_mock, monkeypatch):+ monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "not-an-int")+ with pytest.raises(ValueError, match=r"not an integer"):+ barrier.enqueue(+ [MagicMock()],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ )++ def test_zero_raises(self, barrier, redis_mock, monkeypatch):+ monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "0")+ with pytest.raises(ValueError, match=r"positive integer"):+ barrier.enqueue(+ [MagicMock()],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ )++ def test_negative_raises(self, barrier, redis_mock, monkeypatch):+ monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "-1")+ with pytest.raises(ValueError, match=r"positive integer"):+ barrier.enqueue(+ [MagicMock()],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ )+++# --- Mid-loop dispatch failure ---+++class TestDispatchFailureCleanup:+ """If ``apply_async`` raises mid-fan-out, partially-dispatched+ tasks' links would otherwise corrupt the counter and the orphan+ state would leak. The barrier must DEL its keys before re-raising+ so the in-flight links' DECR returns negative → Lua ``< 0``+ branch cleanly abandons the execution.+ """++ def test_mid_loop_dispatch_failure_cleans_up_keys(+ self, barrier, redis_mock+ ):+ """Task K of N raises during ``apply_async`` — barrier must+ DEL both keys before propagating the exception."""+ h1 = MagicMock(name="h1")+ h2 = MagicMock(name="h2")+ h3_failing = MagicMock(name="h3_failing")+ h3_failing.clone.return_value.apply_async.side_effect = (+ RuntimeError("broker outage")+ )++ with pytest.raises(RuntimeError, match=r"broker outage"):+ barrier.enqueue(+ [h1, h2, h3_failing],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ )++ # Initial SET + EXPIRE happened, then DEL on cleanup.+ # The DEL call should reference BOTH barrier keys.+ delete_calls = redis_mock.delete.call_args_list+ # At least one of the DEL calls must include both barrier keys+ # (the cleanup call), distinguishable from the initial+ # ``delete(_results_key(...))`` stale-list clear.+ cleanup_args = [+ set(call.args) for call in delete_calls+ if len(call.args) == 2+ ]+ assert any(+ {_remaining_key("exec-1"), _results_key("exec-1")} == args+ for args in cleanup_args+ ), (+ f"Expected DEL(remaining, results) in cleanup; saw "+ f"delete calls: {[call.args for call in delete_calls]}"+ )+++# --- link_error: barrier_abort ---+++class TestBarrierAbortLinkError:+ """``link_error`` propagates header-task failures.++ Without this, a failed header task would leave the counter stuck+ above 0 and the execution would hang until TTL. ``barrier_abort``+ DELs the barrier keys + logs the error; the outer task's error+ handler is responsible for the workflow status update.+ """++ def test_abort_registered_under_canonical_name(self):+ """Mixed-version rolling deploys require the link_error task+ to be importable by name on every worker that processes it."""+ from queue_backend import barrier_abort++ assert barrier_abort.name == "barrier_abort"++ def test_link_error_attached_to_each_header_task(self, barrier, redis_mock):+ """Every header task must get ``barrier_abort.s(...)`` as a+ ``.link_error``. Without this, header failures don't+ propagate and the execution hangs."""+ h1 = MagicMock(name="h1")+ h2 = MagicMock(name="h2")+ barrier.enqueue(+ [h1, h2],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ )+ h1.clone.return_value.link_error.assert_called_once()+ h2.clone.return_value.link_error.assert_called_once()++ @patch("queue_backend.redis_barrier._get_redis_client")+ def test_abort_deletes_both_barrier_keys(self, get_redis):+ """``barrier_abort`` cleans up the barrier state on header+ failure — DELs both keys. The first call to ``barrier_abort``+ for a given execution_id wins the SET NX race and proceeds+ with cleanup + log."""+ from queue_backend import barrier_abort++ redis_client = MagicMock(name="redis")+ # ``set(...nx=True)`` returns True on first SET — this abort wins.+ redis_client.set.return_value = True+ get_redis.return_value = redis_client++ barrier_abort.run(+ MagicMock(name="request"),+ RuntimeError("header task crashed"),+ "traceback string",+ execution_id="exec-42",+ )++ redis_client.delete.assert_called_once_with(+ _remaining_key("exec-42"),+ _results_key("exec-42"),+ )++ @patch("queue_backend.redis_barrier._get_redis_client")+ def test_concurrent_aborts_deduplicate(self, get_redis):+ """N concurrent header-task failures attach the same+ ``link_error`` → N ``barrier_abort`` executions for one+ logical execution failure. Without dedup that would produce+ N ``logger.error`` calls and N Sentry events. The SET NX+ lock makes the first abort win; subsequent aborts early-exit+ silently."""+ from queue_backend import barrier_abort++ redis_client = MagicMock(name="redis")+ # First call: SET NX returns True (first abort wins).+ # Second call: SET NX returns False (lock already exists).+ redis_client.set.side_effect = [True, False]+ get_redis.return_value = redis_client++ result_1 = barrier_abort.run(+ MagicMock(name="request"),+ RuntimeError("header task 1 crashed"),+ "traceback string",+ execution_id="exec-42",+ )+ result_2 = barrier_abort.run(+ MagicMock(name="request"),+ RuntimeError("header task 2 crashed"),+ "traceback string",+ execution_id="exec-42",+ )++ # First abort completed cleanup; second deduplicated.+ assert result_1["status"] == "aborted"+ assert result_2["status"] == "deduplicated"++ # DELETE called exactly ONCE — only the winning abort cleans+ # up. The deduplicated path early-exits before DELETE.+ redis_client.delete.assert_called_once_with(+ _remaining_key("exec-42"),+ _results_key("exec-42"),+ )++ @patch("queue_backend.redis_barrier._get_redis_client")+ def test_abort_uses_setnx_with_ttl(self, get_redis):+ """The SET NX lock has an EX (TTL) — the lock key auto-+ cleans alongside the barrier keys instead of leaking+ forever on a failed execution."""+ from queue_backend import barrier_abort+ from queue_backend.redis_barrier import _KEY_TTL_DEFAULT_SECONDS++ redis_client = MagicMock(name="redis")+ redis_client.set.return_value = True+ get_redis.return_value = redis_client++ barrier_abort.run(+ MagicMock(name="request"),+ RuntimeError("crash"),+ "traceback",+ execution_id="exec-1",+ )++ # SET called with nx=True + ex=<ttl>.+ redis_client.set.assert_called_once()+ call = redis_client.set.call_args+ assert call.args[0] == "barrier:abort_lock:exec-1"+ assert call.kwargs.get("nx") is True+ assert call.kwargs.get("ex") == _KEY_TTL_DEFAULT_SECONDS++ @patch("queue_backend.redis_barrier._get_redis_client")+ def test_abort_delete_failure_releases_lock(self, get_redis):+ """If SET NX succeeds (this abort wins) but the subsequent+ DELETE then raises (mid-abort Redis blip), the lock MUST be+ released — otherwise sibling aborts hit the dedup early-exit+ while the barrier keys are still alive, in-flight successful+ tasks DECR to 0, and the callback fires with partial results+ (silent-success-on-mid-abort-failure).++ Same correctness class as the stale-lock-on-retry bug,+ reached via a different path: mid-abort failure instead of+ cross-execution stale state.+ """+ from queue_backend import barrier_abort+ from queue_backend.redis_barrier import _abort_lock_key++ redis_client = MagicMock(name="redis")+ # SET NX returns True (this abort wins), DELETE then raises.+ redis_client.set.return_value = True+ # Two delete calls expected: the barrier-keys DEL that raises,+ # then the lock-release DEL.+ redis_client.delete.side_effect = [+ ConnectionError("redis blip mid-abort"), # first DELETE raises+ 1, # lock-release DEL succeeds (returns count of deleted keys)+ ]+ get_redis.return_value = redis_client++ with pytest.raises(ConnectionError, match=r"redis blip mid-abort"):+ barrier_abort.run(+ MagicMock(name="request"),+ RuntimeError("header task crashed"),+ "traceback",+ execution_id="exec-1",+ )++ # First DELETE was the barrier-keys cleanup; SECOND DELETE was+ # the lock release (so siblings can re-acquire).+ delete_calls = redis_client.delete.call_args_list+ assert len(delete_calls) == 2, (+ f"Expected 2 delete calls (barrier keys + lock release); "+ f"got {len(delete_calls)}: {delete_calls}"+ )+ # The second call must target the abort_lock specifically.+ assert delete_calls[1].args == (_abort_lock_key("exec-1"),), (+ f"Expected lock-release DELETE; got args={delete_calls[1].args!r}"+ )+++# --- Retry with reused execution_id: stale abort_lock cleanup ---+++class TestAbortLockRetryCleanup:+ """A retry that reuses ``execution_id`` must not be poisoned by a+ stale ``abort_lock`` left behind by the previous failed run.++ Failure mode without the fix: execution A fails → ``barrier_abort``+ writes ``abort_lock:foo`` (TTL 6h) → within 6h, execution B retries+ with ``exec_id=foo`` → ``enqueue`` resets remaining/results but+ abort_lock survives → if any header in B fails, its ``barrier_abort``+ hits the stale lock and early-exits without DELing B's keys →+ in-flight successful tasks DECR ``remaining`` to 0 → callback fires+ with partial results → failed retry silently masked as successful.++ Fix: ``enqueue`` DELs the abort_lock alongside the results list at+ setup time, so a retry starts with a clean slate.+ """++ def test_enqueue_clears_stale_abort_lock(self, barrier, redis_mock):+ """``enqueue`` must DEL the abort_lock key at setup so a+ stale lock from a previous (failed) execution with the same+ execution_id doesn't poison the retry's abort path."""+ from queue_backend.redis_barrier import _abort_lock_key++ barrier.enqueue(+ [MagicMock(name="h1")],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-foo"},+ callback_queue="q",+ app_instance=MagicMock(),+ )+ # The setup DEL must include the abort_lock key (alongside+ # results) — otherwise a stale lock from a prior run with the+ # same execution_id survives into this enqueue.+ redis_mock.delete.assert_called_once_with(+ _results_key("exec-foo"),+ _abort_lock_key("exec-foo"),+ )+++# --- Process-local Redis client caching ---+++class TestRedisClientSingleton:+ """``_get_redis_client()`` returns a process-cached client so the+ high-frequency ``barrier_decr_and_check`` / ``barrier_abort``+ invocations reuse the same ``ConnectionPool`` rather than spinning+ up a fresh pool (and TCP handshakes) per task. At 1000-task+ fan-out the difference is 1000 vs 1 pool initialisation.+ """++ def test_get_redis_client_returns_same_instance(self):+ """Repeated calls within one process must return the same+ ``Redis`` instance — guarantees connection-pool reuse."""+ from queue_backend import redis_barrier++ # Reset the cache so this test exercises the lazy-init path.+ redis_barrier._redis_client_singleton = None+ try:+ with patch(+ "queue_backend.redis_barrier.create_redis_client"+ ) as factory:+ factory.return_value = MagicMock(name="redis_client")+ client_a = redis_barrier._get_redis_client()+ client_b = redis_barrier._get_redis_client()+ client_c = redis_barrier._get_redis_client()++ # All three calls return the same object.+ assert client_a is client_b is client_c+ # ``create_redis_client`` factory called exactly ONCE —+ # subsequent calls hit the cache.+ assert factory.call_count == 1+ finally:+ # Reset for downstream tests.+ redis_barrier._redis_client_singleton = None++ def test_falls_back_to_canonical_redis_prefix_for_sentinel_ssl_inheritance(+ self, monkeypatch+ ):+ """When no ``WORKER_BARRIER_REDIS_HOST`` is set, the barrier+ MUST use the canonical ``REDIS_`` prefix directly so it+ inherits the FULL config including ``REDIS_SENTINEL_MODE`` and+ ``REDIS_SSL`` — not just HOST/PORT/PASSWORD/USER/DB.++ Without this fix, a Sentinel-backed deployment that relies on+ the documented fallback would connect standalone (Sentinel+ mode is per-prefix in ``create_redis_client`` and doesn't+ cross-fall-back), failing every execution at flag-flip time."""+ from queue_backend import redis_barrier++ # Ensure no barrier-specific HOST is set.+ monkeypatch.delenv("WORKER_BARRIER_REDIS_HOST", raising=False)+ redis_barrier._redis_client_singleton = None+ try:+ with patch(+ "queue_backend.redis_barrier.create_redis_client"+ ) as factory:+ factory.return_value = MagicMock(name="redis_client")+ redis_barrier._get_redis_client()++ # The factory MUST be called with env_prefix="REDIS_" so+ # SENTINEL_MODE/SSL inherit from the canonical instance.+ factory.assert_called_once()+ assert factory.call_args.kwargs.get("env_prefix") == "REDIS_", (+ f"Expected env_prefix='REDIS_' when no barrier-specific "+ f"HOST is set (so Sentinel/SSL inherit); got "+ f"env_prefix={factory.call_args.kwargs.get('env_prefix')!r}"+ )+ finally:+ redis_barrier._redis_client_singleton = None++ def test_uses_barrier_prefix_when_dedicated_host_set(self, monkeypatch):+ """When ``WORKER_BARRIER_REDIS_HOST`` IS set, the operator has+ opted into a dedicated barrier Redis and is responsible for+ the full barrier-prefix config (HOST + PORT + ... ++ SENTINEL_MODE + SSL)."""+ from queue_backend import redis_barrier++ monkeypatch.setenv("WORKER_BARRIER_REDIS_HOST", "dedicated-redis")+ redis_barrier._redis_client_singleton = None+ try:+ with patch(+ "queue_backend.redis_barrier.create_redis_client"+ ) as factory:+ factory.return_value = MagicMock(name="redis_client")+ redis_barrier._get_redis_client()++ assert (+ factory.call_args.kwargs.get("env_prefix")+ == "WORKER_BARRIER_REDIS_"+ ), (+ f"Expected env_prefix='WORKER_BARRIER_REDIS_' when "+ f"dedicated HOST is set; got "+ f"env_prefix={factory.call_args.kwargs.get('env_prefix')!r}"+ )+ finally:+ redis_barrier._redis_client_singleton = None+++if __name__ == "__main__":+ pytest.main([__file__, "-v"])