LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-4078 [MISC] Remove the Celery execution transport from the workers, backend and SDK (#2284)
ali6154891fdcc5
diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.pyindex 500c6d908..f0f3d16fb 100644--- a/backend/pg_queue/models.py+++ b/backend/pg_queue/models.py@@ -257,12 +257,11 @@ class PgBatchDedup(models.Model):class PgBarrierState(models.Model):- """Per-execution fan-in barrier state for ``PgBarrier`` (the Postgres- ``WORKER_BARRIER_BACKEND``).+ """Per-execution fan-in barrier state for ``PgBarrier``.- One row per in-flight barrier (keyed by ``execution_id``). The worker-side- ``barrier_pg_decr_and_check`` link task atomically decrements ``remaining``- and appends to ``results`` in a single ``UPDATE … RETURNING``; the task that+ One row per in-flight barrier (keyed by ``execution_id``). Each header task+ atomically decrements ``remaining`` in-body and appends to ``results`` in a+ single ``UPDATE … RETURNING``; the task thatdrives ``remaining`` to 0 dispatches the aggregating callback and deletes therow. A header-task failure aborts the barrier by deleting the row outright(``DELETE … RETURNING`` — atomic claim+teardown), so the callback can never@@ -541,8 +540,7 @@ class PgOrchestrationClaim(models.Model):``organization_id`` is stamped below — the reaper needs it for the org-scopedstatus/mark API, exactly like ``pg_barrier_state``.- Only written on the PG transport (``is_pg_transport``) — the Celery path never- touches it. Managed=True / generated migration, extension-free — same posture+ Managed=True / generated migration, extension-free — same postureas the siblings."""diff --git a/backend/pg_queue/tests/test_producer.py b/backend/pg_queue/tests/test_producer.pyindex 246855a95..38ee9cc46 100644--- a/backend/pg_queue/tests/test_producer.py+++ b/backend/pg_queue/tests/test_producer.py@@ -26,7 +26,7 @@ class TestEnqueueTask:task_name="async_execute_bin",queue="celery_api_deployments",args=["org", "wf", "exec"],- kwargs={"transport": "pg_queue"},+ kwargs={"use_file_history": True},org_id="org",priority=5,fairness={@@ -44,7 +44,7 @@ class TestEnqueueTask:assert msg["task_name"] == "async_execute_bin"assert msg["queue"] == "celery_api_deployments"assert msg["args"] == ["org", "wf", "exec"]- assert msg["kwargs"] == {"transport": "pg_queue"}+ assert msg["kwargs"] == {"use_file_history": True}assert msg["fairness"]["workload_type"] == "api"def test_uuid_args_kwargs_are_json_coerced(self):diff --git a/backend/workflow_manager/internal_api_views.py b/backend/workflow_manager/internal_api_views.pyindex c54199f16..15559a71c 100644--- a/backend/workflow_manager/internal_api_views.py+++ b/backend/workflow_manager/internal_api_views.py@@ -18,7 +18,7 @@ from rest_framework.decorators import api_viewfrom rest_framework.response import Responsefrom tool_instance_v2.models import ToolInstance-from unstract.core.data_models import WorkflowTransport+from unstract.core.data_models import LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUEfrom workflow_manager.workflow_v2.enums import ExecutionStatusfrom workflow_manager.workflow_v2.models import Workflow, WorkflowExecution@@ -326,9 +326,8 @@ def create_workflow_execution(request):"execution_id": str(execution.id),"status": execution.status,"execution_log_id": execution.execution_log_id, # Return for workers to use- # Wire field the workers still branch on. PG is the only- # transport now; the field goes when those branches do.- "transport": WorkflowTransport.PG_QUEUE.value,+ # Rolling-deploy shim: see LEGACY_TRANSPORT_KEY.+ LEGACY_TRANSPORT_KEY: LEGACY_TRANSPORT_VALUE,})diff --git a/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py b/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.pyindex 3ee117480..b51bb85e7 100644--- a/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py+++ b/backend/workflow_manager/workflow_v2/tests/test_dispatch_orchestrator.py@@ -21,7 +21,7 @@ class TestDispatchOrchestratorTask:task_id = WorkflowHelper._dispatch_orchestrator_task(queue=CeleryQueue.CELERY_API_DEPLOYMENTS,args=["a"],- kwargs={"transport": "pg_queue"},+ kwargs={"use_file_history": True},org_schema="org1",)# bare msg_id, no "pg:" prefix — matches the worker PgDispatchHandle.iddiff --git a/backend/workflow_manager/workflow_v2/tests/test_legacy_transport_shim.py b/backend/workflow_manager/workflow_v2/tests/test_legacy_transport_shim.pynew file mode 100644index 000000000..3d68a1dbb--- /dev/null+++ b/backend/workflow_manager/workflow_v2/tests/test_legacy_transport_shim.py@@ -0,0 +1,85 @@+"""Rolling-deploy shim for UN-4078.++Nothing reads the ``transport`` key any more, but a pre-UN-4078 worker treats an+absent key as "celery" and publishes to a RabbitMQ with no consumers. The backend+must keep writing it for one release; delete this test with the writes.++DB-free: the model, context, and ``time.sleep`` are mocked.+"""++from unittest.mock import MagicMock, patch++import json++from django.test import RequestFactory++from workflow_manager.internal_api_views import create_workflow_execution+from workflow_manager.workflow_v2.enums import ExecutionStatus+from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper++_MOD = "workflow_manager.workflow_v2.workflow_helper"+_VIEWS = "workflow_manager.internal_api_views"+++class TestOrchestratorPayloadCarriesLegacyTransport:+ def test_dispatch_kwargs_include_pg_queue_transport(self):+ exec_row = MagicMock(status=ExecutionStatus.EXECUTING.value)++ with (+ patch(f"{_MOD}.UserContext") as user_ctx,+ patch(f"{_MOD}.StateStore") as state_store,+ patch(f"{_MOD}.time"),+ patch(f"{_MOD}.WorkflowExecution") as wf_exec,+ patch.object(+ WorkflowHelper, "_dispatch_orchestrator_task", return_value="1"+ ) as dispatch,+ patch.object(WorkflowHelper, "_record_dispatch_handle"),+ ):+ user_ctx.get_organization_identifier.return_value = "org1"+ state_store.get.return_value = None+ wf_exec.objects.get.return_value = exec_row++ WorkflowHelper.execute_workflow_async(+ workflow_id="wf-1",+ execution_id="exec-1",+ hash_values_of_files={},+ )++ dispatch.assert_called_once()+ assert dispatch.call_args.kwargs["kwargs"]["transport"] == "pg_queue"+++class TestCreateWorkflowExecutionResponseCarriesLegacyTransport:+ """The scheduler reads this response body.++ A pre-UN-4078 scheduler does ``workflow_execution.get("transport",+ DEFAULT_WORKFLOW_TRANSPORT)`` against exactly this dict, so dropping the key+ sends it down the Celery branch.+ """++ def test_response_includes_pg_queue_transport(self):+ execution = MagicMock(+ id="exec-1",+ status=ExecutionStatus.PENDING.value,+ execution_log_id="log-1",+ )++ with (+ patch(f"{_VIEWS}.Organization") as organization,+ patch(f"{_VIEWS}.Workflow") as workflow,+ patch(f"{_VIEWS}.WorkflowExecution") as wf_exec,+ ):+ organization.objects.get.return_value = MagicMock()+ workflow.objects.get.return_value = MagicMock()+ wf_exec.objects.create.return_value = execution++ request = RequestFactory().post(+ "/internal/v1/workflow-execution/",+ data=json.dumps({"workflow_id": "wf-1", "total_files": 1}),+ content_type="application/json",+ HTTP_X_ORGANIZATION_ID="org-1",+ )+ response = create_workflow_execution(request)++ assert response.data["transport"] == "pg_queue"+ assert response.data["execution_id"] == "exec-1"diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex da5ad380c..2c926d0dc 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -27,7 +27,11 @@ from utils.local_context import StateStorefrom utils.user_context import UserContextfrom backend.celery_service import app as celery_app-from unstract.core.data_models import WorkflowTransport, WorkloadType+from unstract.core.data_models import (+ LEGACY_TRANSPORT_KEY,+ LEGACY_TRANSPORT_VALUE,+ WorkloadType,+)from unstract.workflow_execution.enums import LogStagefrom workflow_manager.endpoint_v2.destination import DestinationConnectorfrom workflow_manager.endpoint_v2.dto import FileHash@@ -71,7 +75,11 @@ EXECUTION_EXCLUDED_PARAMS = {"hitl_queue_name","hitl_packet_id","custom_data",- "transport",+ # Still carried on the dispatch payload as a rolling-deploy shim+ # (LEGACY_TRANSPORT_KEY), so it must never reach the legacy+ # ``execute_workflow`` signature, where it would raise as an unexpected kwarg.+ # Named through the constant so the shim's removal checklist finds this site.+ LEGACY_TRANSPORT_KEY,}@@ -620,9 +628,8 @@ class WorkflowHelper:"hitl_queue_name": hitl_queue_name,"hitl_packet_id": hitl_packet_id,"custom_data": custom_data,- # Wire field the workers still branch on. PG is the only- # transport now; the field goes when those branches do.- "transport": WorkflowTransport.PG_QUEUE.value,+ # Rolling-deploy shim: see LEGACY_TRANSPORT_KEY.+ LEGACY_TRANSPORT_KEY: LEGACY_TRANSPORT_VALUE,}dispatch_handle = cls._dispatch_orchestrator_task(queue=queue,diff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.pyindex 00073ee2a..4ba675d54 100644--- a/unstract/core/src/unstract/core/data_models.py+++ b/unstract/core/src/unstract/core/data_models.py@@ -200,92 +200,28 @@ class SourceConnectionType(str, Enum):API = "API"-class WorkflowTransport(str, Enum):- """Transport a single workflow execution rides end-to-end.-- The migration unit is the *execution*, not the task: every stage of a- coupled pipeline (async_execute → file-batch fan-out → callback) must run- on one transport, decided once at execution creation and carried in the- task payload (see ``9e-design.md``). ``CELERY`` is the legacy default;- ``PG_QUEUE`` is the bespoke Postgres queue.- """-- CELERY = "celery"- PG_QUEUE = "pg_queue"---# Default transport when none is carried on the payload.+# Rolling-deploy shim (UN-4078). Nothing BRANCHES on this key any more, but it is+# still written for one release, and one site still matches it by name (the+# EXECUTION_EXCLUDED_PARAMS entry below). A pre-UN-4078 worker treats an absent+# ``transport`` as "celery" and publishes the fan-out / callback to RabbitMQ, which+# has no consumers, so during the rollout window an old pod receiving a new+# producer's payload would strand the execution.#-# ⚠️ KNOWN HAZARD, DELIBERATELY NOT FIXED HERE — the safe direction has inverted.-#-# CELERY was the safe fallback while Celery had consumers: a payload that lost the-# field degraded onto a working substrate. With the Celery fleet at zero that is no-# longer true. A payload defaulting to "celery" now selects CeleryChordBarrier in-# `_barrier_for_transport`, skips the PG orchestration-claim and terminal-redelivery-# guards, and publishes the fan-out to a broker nothing drains — and because no PG-# rows are written, neither the reaper nor the undispatched sweep can see the-# stranded execution. Reachable on a rolling deploy, where a not-yet-upgraded-# backend still emits `transport: "celery"`.-#-# Flipping this constant to PG_QUEUE is the right fix and it is a one-line change,-# but it is NOT a one-line consequence: it breaks 19 tests across 6 files-# (test_pg_barrier, test_chord_sites_characterisation, test_barrier,-# test_dispatch_with_callback, test_dispatch_sites_characterisation,-# test_workflow_context_transport), every one of which relies on the implicit-# Celery default — most of them characterising PgBarrier's Celery-link branch,-# which is deferred-deletion code. Rewriting those to state their transport-# explicitly is correct, but it is a behaviour change that deserves its own review-# rather than riding along in a documentation sweep.-#-# Until then the mitigation is that every producer sets the field explicitly-# (workflow_helper.py and internal_api_views.py both hardcode PG_QUEUE), so the-# default is only reachable via version skew.-DEFAULT_WORKFLOW_TRANSPORT = WorkflowTransport.CELERY.value---def normalize_transport(value: object, *, logger: Any = None, context: str = "") -> str:- """Coerce an inbound transport value to a known ``WorkflowTransport`` value.-- The transport crosses untrusted boundaries — a task payload, PG JSONB, an- older backend that omits the field — so a missing/garbage value must never- route an execution onto an unknown substrate. This **fails closed**: an- unrecognized value (``None``, ``"celary"``, ``""``) logs a warning (when a- ``logger`` is given) and falls back to :data:`DEFAULT_WORKFLOW_TRANSPORT`- — see the hazard note on that constant. A recognized value passes through as- its canonical string.-- Coerce-not-raise is deliberate, and differs from how- ``WorkflowContextData.workflow_type`` is validated: ``transport`` has an- explicit safe default by design, so a bad value should degrade rather than- crash the execution. What changed with UN-4046 is WHICH default is safe — see- the note on the constant. A rolling deploy where an older backend still emits- ``"celery"`` is the live case: that value is recognized, so it passes through- here untouched and the guards keyed on it are skipped. This function only- protects the missing/garbage case. ``context`` is an optional suffix for the- log line (e.g. an ``exec:<id>`` tag) to make a version-skew warning traceable.- """- try:- return WorkflowTransport(value).value- except ValueError:- if logger is not None:- logger.warning(- "Unrecognized workflow transport %r%s; falling back to %r",- value,- context,- DEFAULT_WORKFLOW_TRANSPORT,- )- return DEFAULT_WORKFLOW_TRANSPORT---def is_pg_transport(transport: str | None) -> bool:- """True if ``transport`` is the Postgres-queue transport.-- Single source for "what counts as PG transport" — centralises the- ``== WorkflowTransport.PG_QUEUE.value`` comparison scattered across the- worker fan-out / barrier code, and the seam to extend if a second- PG-family transport is ever added.- """- return transport == WorkflowTransport.PG_QUEUE.value+# REMOVAL CHECKLIST for the release after UN-4078, once no pre-UN-4078 worker can+# still be running (`grep -rn LEGACY_TRANSPORT_ ` finds every item but the last):+# 1. the four writes — backend ``WorkflowHelper`` dispatch payload, backend+# ``create_workflow_execution`` response, ``workers/scheduler/tasks.py``+# dispatch kwargs, ``PgBarrier`` callback descriptor+# 2. ``CallbackDescriptor.transport`` — workers/queue_backend/barrier.py+# 3. the ``LEGACY_TRANSPORT_KEY`` entry in ``EXECUTION_EXCLUDED_PARAMS`` —+# backend/workflow_manager/workflow_v2/workflow_helper.py+# 4. these two constants+# 5. the gating tests — workers/tests/test_legacy_transport_shim.py,+# workers/tests/test_pg_barrier.py, the shim cases in+# workers/tests/test_dispatch_sites_characterisation.py, and+# backend/workflow_manager/workflow_v2/tests/test_legacy_transport_shim.py+LEGACY_TRANSPORT_KEY = "transport"+LEGACY_TRANSPORT_VALUE = "pg_queue"class WorkloadType(StrEnum):diff --git a/unstract/sdk1/src/unstract/sdk1/execution/__init__.py b/unstract/sdk1/src/unstract/sdk1/execution/__init__.pyindex fa70c8882..a91602dcb 100644--- a/unstract/sdk1/src/unstract/sdk1/execution/__init__.py+++ b/unstract/sdk1/src/unstract/sdk1/execution/__init__.py@@ -1,5 +1,4 @@from unstract.sdk1.execution.context import ExecutionContext-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.sdk1.execution.executor import BaseExecutorfrom unstract.sdk1.execution.orchestrator import ExecutionOrchestratorfrom unstract.sdk1.execution.registry import ExecutorRegistry@@ -8,7 +7,6 @@ from unstract.sdk1.execution.result import ExecutionResult__all__ = ["BaseExecutor","ExecutionContext",- "ExecutionDispatcher","ExecutionOrchestrator","ExecutionResult","ExecutorRegistry",diff --git a/unstract/sdk1/src/unstract/sdk1/execution/dispatcher.py b/unstract/sdk1/src/unstract/sdk1/execution/dispatcher.pydeleted file mode 100644index 25a5da1f0..000000000--- a/unstract/sdk1/src/unstract/sdk1/execution/dispatcher.py+++ /dev/null@@ -1,316 +0,0 @@-"""Execution dispatcher for sending Celery tasks to executor workers.--The dispatcher is the caller-side component used by both:-- Structure tool Celery task (workflow path)-- PromptStudioHelper (IDE path)--It sends ``execute_extraction`` tasks to the ``executor`` queue.-Three dispatch modes are available:--- ``dispatch()``: Send and block until result (synchronous).-- ``dispatch_async()``: Fire-and-forget, returns task_id for polling.-- ``dispatch_with_callback()``: Fire-and-forget with Celery ``link``- / ``link_error`` callbacks for post-processing.-"""--from __future__ import annotations--import logging-import os-from typing import TYPE_CHECKING, Any--if TYPE_CHECKING:- from celery import Signature- from celery.result import AsyncResult- from unstract.sdk1.execution.context import ExecutionContext--from unstract.sdk1.execution.result import ExecutionResult--logger = logging.getLogger(__name__)--# Constants matching workers/shared/enums values.-# Defined here to avoid an SDK1 → workers package dependency.-_TASK_NAME = "execute_extraction"--# Queue-per-executor prefix. Each executor gets its own Celery queue-# named ``celery_executor_{executor_name}``, derived automatically-# from ``ExecutionContext.executor_name``.-_QUEUE_PREFIX = "celery_executor_"--# Caller-side timeout (seconds) for AsyncResult.get().-# This controls how long the *caller* waits for the executor to-# finish — distinct from the executor worker's-# ``EXECUTOR_TASK_TIME_LIMIT`` which controls how long the-# *worker* allows a task to run.-#-# Resolution order (matches workers convention):-# 1. Explicit ``timeout`` parameter on dispatch()-# 2. ``EXECUTOR_RESULT_TIMEOUT`` env var-# 3. Hardcoded default (3600s)-#-# The default (3600s) is intentionally <= the executor worker's-# ``task_time_limit`` default (also 3600s) so the caller never-# waits longer than the worker allows the task to run.-_DEFAULT_TIMEOUT_ENV = "EXECUTOR_RESULT_TIMEOUT"-_DEFAULT_TIMEOUT = 3600 # 1 hour — matches executor worker default-_NO_CELERY_APP_MSG = "No Celery app configured on ExecutionDispatcher"---class ExecutionDispatcher:- """Dispatches execution to executor worker via Celery task.-- Usage::-- dispatcher = ExecutionDispatcher(celery_app=app)- result = dispatcher.dispatch(context, timeout=120)-- Fire-and-forget::-- task_id = dispatcher.dispatch_async(context)-- Fire-and-forget with callbacks::-- from celery import signature-- task = dispatcher.dispatch_with_callback(- context,- on_success=signature("my_success_task", args=[...], queue="q"),- on_error=signature("my_error_task", args=[...], queue="q"),- )- """-- def __init__(self, celery_app: object | None = None) -> None:- """Initialize the dispatcher.-- Args:- celery_app: A Celery application instance. Required- for dispatching tasks. Can be ``None`` only if- set later via ``celery_app`` attribute.- """- self._app = celery_app-- @staticmethod- def _get_queue(executor_name: str) -> str:- """Derive the Celery queue name from *executor_name*.-- Convention: ``celery_executor_{executor_name}``.- Adding a new executor automatically gets its own queue —- no registry change needed.- """- return f"{_QUEUE_PREFIX}{executor_name}"-- @staticmethod- def _build_send_kwargs(- context: ExecutionContext,- queue: str,- *,- headers: dict[str, Any] | None = None,- link: Signature | None = None,- link_error: Signature | None = None,- task_id: str | None = None,- ) -> dict[str, Any]:- """Common ``send_task`` kwargs builder for the three dispatch modes.-- Optional kwargs are dropped when ``None`` so the resulting call- shape matches the minimum-arity form Celery expects for ``link``- / ``link_error`` handling and so callers that pass no headers- see the same on-wire call as before headers existed.- """- out: dict[str, Any] = {- "args": [context.to_dict()],- "queue": queue,- }- # Treat falsy (None, ``{}``) as drop. An empty header dict is- # never legitimate — ``FairnessKey.as_header()`` always returns- # a populated mapping — so silently forwarding ``{}`` would- # document a producer-side build bug rather than catch it.- if headers:- out["headers"] = headers- if link is not None:- out["link"] = link- if link_error is not None:- out["link_error"] = link_error- if task_id is not None:- out["task_id"] = task_id- return out-- def dispatch(- self,- context: ExecutionContext,- timeout: int | None = None,- headers: dict[str, Any] | None = None,- ) -> ExecutionResult:- """Dispatch context as a Celery task and wait for result.-- Args:- context: ExecutionContext to dispatch.- timeout: Max seconds to wait. ``None`` reads from- the ``EXECUTOR_RESULT_TIMEOUT`` env var,- falling back to 3600s.- headers: Optional Celery message headers (routing metadata- out-of-band of the task body — see- ``queue_backend.fairness.FAIRNESS_HEADER_NAME``- (currently ``"x-fairness-key"``) for the canonical- constant). Caller is responsible for the header schema.- Falsy values (``None``, ``{}``) are dropped — the- downstream ``send_task`` call is invoked without- ``headers=`` so the on-wire shape matches the- no-headers baseline.-- Returns:- ExecutionResult from the executor.-- Raises:- ValueError: If no Celery app is configured.- """- if self._app is None:- raise ValueError(_NO_CELERY_APP_MSG)-- if timeout is None:- timeout = int(os.environ.get(_DEFAULT_TIMEOUT_ENV, _DEFAULT_TIMEOUT))-- queue = self._get_queue(context.executor_name)- logger.info(- "Dispatching execution: executor=%s operation=%s"- " run_id=%s request_id=%s timeout=%ss queue=%s",- context.executor_name,- context.operation,- context.run_id,- context.request_id,- timeout,- queue,- )-- send_kwargs = self._build_send_kwargs(context, queue, headers=headers)- async_result = self._app.send_task(_TASK_NAME, **send_kwargs)- logger.info(- "Task sent: celery_task_id=%s, waiting for result...",- async_result.id,- )-- try:- # disable_sync_subtasks=False: safe because the executor task- # runs on a separate worker pool (worker-v2) — no deadlock- # risk even when dispatch() is called from inside a Django- # Celery task.- result_dict = async_result.get(- timeout=timeout,- disable_sync_subtasks=False,- )- except Exception as exc:- logger.error(- "Dispatch failed: executor=%s operation=%s run_id=%s error=%s",- context.executor_name,- context.operation,- context.run_id,- exc,- )- return ExecutionResult.failure(- error=f"{type(exc).__name__}: {exc}",- )-- return ExecutionResult.from_dict(result_dict)-- def dispatch_async(- self,- context: ExecutionContext,- headers: dict[str, Any] | None = None,- ) -> str:- """Dispatch without waiting. Returns task_id for polling.-- Args:- context: ExecutionContext to dispatch.- headers: Optional Celery message headers (see ``dispatch``).-- Returns:- The Celery task ID (use with ``AsyncResult`` to poll).-- Raises:- ValueError: If no Celery app is configured.- """- if self._app is None:- raise ValueError(_NO_CELERY_APP_MSG)-- queue = self._get_queue(context.executor_name)- logger.info(- "Dispatching async execution: executor=%s "- "operation=%s run_id=%s request_id=%s queue=%s",- context.executor_name,- context.operation,- context.run_id,- context.request_id,- queue,- )-- send_kwargs = self._build_send_kwargs(context, queue, headers=headers)- async_result = self._app.send_task(_TASK_NAME, **send_kwargs)- return async_result.id-- def dispatch_with_callback(- self,- context: ExecutionContext,- on_success: Signature | None = None,- on_error: Signature | None = None,- task_id: str | None = None,- headers: dict[str, Any] | None = None,- ) -> AsyncResult:- """Fire-and-forget dispatch with Celery link callbacks.-- Sends the task to the executor queue and returns immediately.- When the executor task completes, Celery invokes the- ``on_success`` callback (via ``link``). If the executor task- raises an exception, Celery invokes ``on_error`` (via- ``link_error``).-- Args:- context: ExecutionContext to dispatch.- on_success: A Celery ``Signature`` invoked on success.- Receives ``(result_dict,)`` as first positional arg- followed by the signature's own args.- on_error: A Celery ``Signature`` invoked on failure.- Receives ``(failed_task_uuid,)`` as first positional- arg followed by the signature's own args.- task_id: Optional pre-generated Celery task ID. Useful- when the caller needs to know the task ID before- dispatch (e.g. to include it in callback kwargs).- headers: Optional Celery message headers (see ``dispatch``).-- Returns:- The ``AsyncResult`` from ``send_task``. Callers can- use ``.id`` for task tracking but should NOT call- ``.get()`` (that would block, defeating the purpose).-- Raises:- ValueError: If no Celery app is configured.- """- if self._app is None:- raise ValueError(_NO_CELERY_APP_MSG)-- queue = self._get_queue(context.executor_name)- logger.info(- "Dispatching with callback: executor=%s "- "operation=%s run_id=%s request_id=%s "- "on_success=%s on_error=%s queue=%s",- context.executor_name,- context.operation,- context.run_id,- context.request_id,- on_success,- on_error,- queue,- )-- send_kwargs = self._build_send_kwargs(- context,- queue,- headers=headers,- link=on_success,- link_error=on_error,- task_id=task_id,- )- async_result = self._app.send_task(_TASK_NAME, **send_kwargs)- logger.info(- "Task sent with callbacks: celery_task_id=%s",- async_result.id,- )- return async_resultdiff --git a/unstract/sdk1/tests/test_execution.py b/unstract/sdk1/tests/test_execution.pyindex 374734597..1dcb8b18a 100644--- a/unstract/sdk1/tests/test_execution.py+++ b/unstract/sdk1/tests/test_execution.py@@ -5,7 +5,6 @@ import loggingimport osimport tempfilefrom typing import Any, Self-from unittest.mock import MagicMockimport pytestfrom unstract.sdk1.constants import LogLevel, ToolEnv@@ -15,7 +14,6 @@ from unstract.sdk1.execution.context import (ExecutionSource,Operation,)-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.sdk1.execution.executor import BaseExecutorfrom unstract.sdk1.execution.orchestrator import ExecutionOrchestratorfrom unstract.sdk1.execution.registry import ExecutorRegistry@@ -585,362 +583,6 @@ class TestExecutionOrchestrator:assert result.error == "LLM rate limited"-# ---- Phase 1F: ExecutionDispatcher -------class TestExecutionDispatcher:- """Tests for ExecutionDispatcher (mocked Celery)."""-- def _make_context(self, **overrides: Any) -> ExecutionContext: # noqa: ANN401- defaults: dict[str, Any] = {- "executor_name": "legacy",- "operation": "extract",- "run_id": "run-1",- "execution_source": "tool",- "request_id": "req-1",- }- defaults.update(overrides)- return ExecutionContext(**defaults)-- def _make_mock_app(- self,- result_dict: dict[str, Any] | None = None,- side_effect: Exception | None = None,- task_id: str = "celery-task-123",- ) -> MagicMock:- """Create a mock Celery app with send_task configured."""- mock_app = MagicMock()- mock_async_result = MagicMock()- mock_async_result.id = task_id-- if side_effect is not None:- mock_async_result.get.side_effect = side_effect- else:- mock_async_result.get.return_value = (- result_dict- if result_dict is not None- else {"success": True, "data": {}, "metadata": {}}- )-- mock_app.send_task.return_value = mock_async_result- return mock_app-- def test_dispatch_sends_task_and_returns_result(- self: Self,- ) -> None:- """dispatch() sends task to executor queue and returns result."""- result_dict = {- "success": True,- "data": {"extracted_text": "hello"},- "metadata": {},- }- mock_app = self._make_mock_app(result_dict=result_dict)- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- result = dispatcher.dispatch(ctx, timeout=60)-- assert result.success is True- assert result.data == {"extracted_text": "hello"}-- # Verify send_task was called correctly- mock_app.send_task.assert_called_once_with(- "execute_extraction",- args=[ctx.to_dict()],- queue="celery_executor_legacy",- )- mock_app.send_task.return_value.get.assert_called_once_with(- timeout=60, disable_sync_subtasks=False- )-- def test_dispatch_uses_default_timeout(self: Self) -> None:- """dispatch() without timeout uses default (3600s)."""- mock_app = self._make_mock_app()- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- dispatcher.dispatch(ctx)-- mock_app.send_task.return_value.get.assert_called_once_with(- timeout=3600, disable_sync_subtasks=False- )-- def test_dispatch_timeout_from_env(- self: Self, monkeypatch: pytest.MonkeyPatch- ) -> None:- """dispatch() reads timeout from EXECUTOR_RESULT_TIMEOUT env."""- monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "120")- mock_app = self._make_mock_app()- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- dispatcher.dispatch(ctx)-- mock_app.send_task.return_value.get.assert_called_once_with(- timeout=120, disable_sync_subtasks=False- )-- def test_dispatch_explicit_timeout_overrides_env(- self: Self, monkeypatch: pytest.MonkeyPatch- ) -> None:- """Explicit timeout parameter overrides env var."""- monkeypatch.setenv("EXECUTOR_RESULT_TIMEOUT", "120")- mock_app = self._make_mock_app()- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- dispatcher.dispatch(ctx, timeout=30)-- mock_app.send_task.return_value.get.assert_called_once_with(- timeout=30, disable_sync_subtasks=False- )-- def test_dispatch_timeout_returns_failure(- self: Self,- ) -> None:- """TimeoutError from AsyncResult.get() is wrapped in failure."""- mock_app = self._make_mock_app(side_effect=TimeoutError("Task timed out"))- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- result = dispatcher.dispatch(ctx, timeout=1)-- assert result.success is False- assert "TimeoutError" in result.error-- def test_dispatch_generic_exception_returns_failure(- self: Self,- ) -> None:- """Any exception from AsyncResult.get() becomes a failure."""- mock_app = self._make_mock_app(side_effect=RuntimeError("broker down"))- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- result = dispatcher.dispatch(ctx, timeout=10)-- assert result.success is False- assert "RuntimeError" in result.error- assert "broker down" in result.error-- def test_dispatch_async_returns_task_id(self: Self) -> None:- """dispatch_async() returns the Celery task ID."""- mock_app = self._make_mock_app(task_id="task-xyz-789")- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- task_id = dispatcher.dispatch_async(ctx)-- assert task_id == "task-xyz-789"- mock_app.send_task.assert_called_once_with(- "execute_extraction",- args=[ctx.to_dict()],- queue="celery_executor_legacy",- )-- def test_dispatch_no_app_raises_value_error(- self: Self,- ) -> None:- """dispatch() without celery_app raises ValueError."""- dispatcher = ExecutionDispatcher(celery_app=None)- ctx = self._make_context()-- with pytest.raises(ValueError, match="No Celery app"):- dispatcher.dispatch(ctx)-- def test_dispatch_async_no_app_raises_value_error(- self: Self,- ) -> None:- """dispatch_async() without celery_app raises ValueError."""- dispatcher = ExecutionDispatcher(celery_app=None)- ctx = self._make_context()-- with pytest.raises(ValueError, match="No Celery app"):- dispatcher.dispatch_async(ctx)-- def test_dispatch_failure_result_from_executor(- self: Self,- ) -> None:- """Executor failure is deserialized correctly."""- result_dict = {- "success": False,- "data": {},- "metadata": {},- "error": "LLM adapter timeout",- }- mock_app = self._make_mock_app(result_dict=result_dict)- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- result = dispatcher.dispatch(ctx, timeout=60)-- assert result.success is False- assert result.error == "LLM adapter timeout"-- def test_dispatch_context_serialized_correctly(- self: Self,- ) -> None:- """The full ExecutionContext is serialized in the task args."""- mock_app = self._make_mock_app()- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context(- executor_name="agentic_table",- operation="agentic_extraction",- organization_id="org-42",- executor_params={"schema": {"name": "str"}},- )-- dispatcher.dispatch(ctx, timeout=60)-- sent_args = mock_app.send_task.call_args- context_dict = sent_args[1]["args"][0]-- assert context_dict["executor_name"] == "agentic_table"- assert context_dict["operation"] == "agentic_extraction"- assert context_dict["organization_id"] == "org-42"- assert context_dict["executor_params"] == {"schema": {"name": "str"}}-- # ---- Phase 5A: dispatch_with_callback ------ def test_dispatch_with_callback_sends_link_and_link_error(- self: Self,- ) -> None:- """dispatch_with_callback() passes on_success as link, on_error as link_error."""- mock_app = self._make_mock_app(task_id="cb-task-001")- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- on_success = MagicMock(name="on_success_sig")- on_error = MagicMock(name="on_error_sig")-- result = dispatcher.dispatch_with_callback(- ctx, on_success=on_success, on_error=on_error- )-- assert result.id == "cb-task-001"- mock_app.send_task.assert_called_once_with(- "execute_extraction",- args=[ctx.to_dict()],- queue="celery_executor_legacy",- link=on_success,- link_error=on_error,- )-- def test_dispatch_with_callback_success_only(- self: Self,- ) -> None:- """dispatch_with_callback() with only on_success omits link_error."""- mock_app = self._make_mock_app(task_id="cb-task-002")- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- on_success = MagicMock(name="on_success_sig")-- dispatcher.dispatch_with_callback(ctx, on_success=on_success)-- call_kwargs = mock_app.send_task.call_args- assert call_kwargs[1]["link"] is on_success- assert "link_error" not in call_kwargs[1]-- def test_dispatch_with_callback_error_only(- self: Self,- ) -> None:- """dispatch_with_callback() with only on_error omits link."""- mock_app = self._make_mock_app(task_id="cb-task-003")- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- on_error = MagicMock(name="on_error_sig")-- dispatcher.dispatch_with_callback(ctx, on_error=on_error)-- call_kwargs = mock_app.send_task.call_args- assert "link" not in call_kwargs[1]- assert call_kwargs[1]["link_error"] is on_error-- def test_dispatch_with_callback_no_callbacks(- self: Self,- ) -> None:- """dispatch_with_callback() with no callbacks sends plain task."""- mock_app = self._make_mock_app(task_id="cb-task-004")- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- result = dispatcher.dispatch_with_callback(ctx)-- assert result.id == "cb-task-004"- call_kwargs = mock_app.send_task.call_args- assert "link" not in call_kwargs[1]- assert "link_error" not in call_kwargs[1]-- def test_dispatch_with_callback_returns_async_result(- self: Self,- ) -> None:- """dispatch_with_callback() returns the AsyncResult object (not just task_id)."""- mock_app = self._make_mock_app(task_id="cb-task-005")- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- result = dispatcher.dispatch_with_callback(ctx)-- # Returns the full AsyncResult, not just the id string- assert result is mock_app.send_task.return_value- assert result.id == "cb-task-005"-- def test_dispatch_with_callback_no_app_raises_value_error(- self: Self,- ) -> None:- """dispatch_with_callback() without celery_app raises ValueError."""- dispatcher = ExecutionDispatcher(celery_app=None)- ctx = self._make_context()-- with pytest.raises(ValueError, match="No Celery app"):- dispatcher.dispatch_with_callback(ctx)-- def test_dispatch_with_callback_context_serialized(- self: Self,- ) -> None:- """dispatch_with_callback() serializes context correctly."""- mock_app = self._make_mock_app()- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context(- operation="answer_prompt",- executor_params={"prompt_key": "p1"},- )-- dispatcher.dispatch_with_callback(ctx, on_success=MagicMock())-- sent_args = mock_app.send_task.call_args- context_dict = sent_args[1]["args"][0]- assert context_dict["operation"] == "answer_prompt"- assert context_dict["executor_params"] == {"prompt_key": "p1"}-- def test_dispatch_with_callback_custom_task_id(- self: Self,- ) -> None:- """dispatch_with_callback() passes custom task_id to send_task."""- mock_app = self._make_mock_app(task_id="pre-gen-id-123")- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- dispatcher.dispatch_with_callback(ctx, task_id="pre-gen-id-123")-- call_kwargs = mock_app.send_task.call_args- assert call_kwargs[1]["task_id"] == "pre-gen-id-123"-- def test_dispatch_with_callback_no_task_id_omits_kwarg(- self: Self,- ) -> None:- """dispatch_with_callback() without task_id doesn't pass task_id."""- mock_app = self._make_mock_app()- dispatcher = ExecutionDispatcher(celery_app=mock_app)- ctx = self._make_context()-- dispatcher.dispatch_with_callback(ctx)-- call_kwargs = mock_app.send_task.call_args- assert "task_id" not in call_kwargs[1]--# ---- Phase 1G: ExecutorToolShim ----# Note: ExecutorToolShim lives in workers/executor/ but the tests# import it directly via sys.path manipulation since the workersdiff --git a/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.pyindex 3fc3a2017..fcfad8836 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/executor_rpc.py@@ -49,6 +49,18 @@ EXECUTE_TASK = "execute_extraction"# Mirror the SDK's queue-per-executor convention so the PG executor queue name# matches the Celery one (the worker-pg-executor consumer subscribes to these).QUEUE_PREFIX = "celery_executor_"+++def queue_for_executor(executor_name: str) -> str:+ """The queue an executor's dispatches land on.++ One definition for the three dispatch sites below (and for the tests, which+ import this rather than re-deriving the rule) so the wire contract+ ``worker-pg-executor`` subscribes to cannot drift between them.+ """+ return f"{QUEUE_PREFIX}{executor_name}"++# Caller-side wait default — mirrors the SDK dispatcher (EXECUTOR_RESULT_TIMEOUT env,# else 3600s) so a PG-routed caller waits exactly as long as a Celery one.DEFAULT_TIMEOUT_ENV = "EXECUTOR_RESULT_TIMEOUT"@@ -152,7 +164,7 @@ class PgExecutionDispatcher:"""timeout = _resolve_timeout(timeout)reply_key = str(uuid.uuid4())- queue = f"{QUEUE_PREFIX}{context.executor_name}"+ queue = queue_for_executor(context.executor_name)org = str(getattr(context, "organization_id", "") or "")try:self._transport.enqueue(@@ -249,7 +261,7 @@ class PgExecutionDispatcher:failure is observable even if the caller swallows it."""task_id = str(uuid.uuid4())- queue = f"{QUEUE_PREFIX}{context.executor_name}"+ queue = queue_for_executor(context.executor_name)org = str(getattr(context, "organization_id", "") or "")try:self._transport.enqueue(@@ -294,7 +306,7 @@ class PgExecutionDispatcher:prompt-studio views do, in their own try/except)."""task_id = task_id or str(uuid.uuid4())- queue = f"{QUEUE_PREFIX}{context.executor_name}"+ queue = queue_for_executor(context.executor_name)org = str(getattr(context, "organization_id", "") or "")success_spec = signature_to_continuation(on_success)error_spec = signature_to_continuation(on_error)diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.pyindex a07ef7b7b..4d4b56b3b 100644--- a/workers/api-deployment/tasks.py+++ b/workers/api-deployment/tasks.py@@ -27,12 +27,9 @@ from shared.workflow.execution.tool_validation import validate_workflow_tool_insfrom worker import appfrom unstract.core.data_models import (- DEFAULT_WORKFLOW_TRANSPORT,ExecutionStatus,FileHashData,WorkerFileData,- is_pg_transport,- normalize_transport,)from unstract.core.worker_models import ApiDeploymentResultStatus@@ -422,15 +419,6 @@ def _run_workflow_api(WorkflowHelper.process_input_files() methods."""total_files = len(hash_values_of_files)- # Resolve transport ONCE, up front from the authoritative kwargs source, so- # the failure handler can rely on it even if orchestration fails before the- # fan-out (the PR-1 seam; the fan-out honours it → PG path = PgBarrier).- # Fail-closed to celery on any unrecognised value.- transport = normalize_transport(- kwargs.get("transport", DEFAULT_WORKFLOW_TRANSPORT),- logger=logger,- context=f" [exec:{execution_id}]",- )# TOOL VALIDATION: Validate tool instances before API workflow orchestration# Get workflow execution context to retrieve tool instances@@ -686,20 +674,15 @@ def _run_workflow_api(file_processing_callback_queue = _get_callback_queue_name_api()# Route through ``WorkflowOrchestrationUtils.create_chord_execution``- # (which delegates to ``CeleryChordBarrier`` under the hood) instead of- # calling ``chord(...)`` directly. Same Celery semantics, but the- # ``Barrier`` abstraction lets a future Phase 6b swap to- # ``RedisDecrBarrier`` behind a flag without touching this call site.- # The header tasks + callback now carry the fairness slot for- # downstream PG Queue routing — closes the chord-fairness gap noted- # in Phase 5.1.- # Hoist FairnessKey to a local so the chord path and the+ # rather than reaching for a barrier directly, so both fan-out call+ # sites share one dispatch point. The header tasks + callback carry the+ # fairness slot for downstream routing.+ # Hoist FairnessKey to a local so the fan-out path and the# zero-batch fallback can't drift apart on the slot's shape.api_fairness = FairnessKey(org_id=str(schema_name),workload_type=WorkloadType.API,)- # transport was resolved up front (see top of _run_workflow_api).result = WorkflowOrchestrationUtils.create_chord_execution(batch_tasks=batch_tasks,callback_task_name="process_batch_callback_api",@@ -711,7 +694,6 @@ def _run_workflow_api(callback_queue=file_processing_callback_queue,app_instance=app,fairness=api_fairness,- transport=transport,)if result is None:@@ -810,36 +792,27 @@ def _run_workflow_api(except Exception as e:error_message = f"Error while processing files: {str(e)}"- # On PG, surface the error to the UI + reconcile file counters so a failed- # run reads "N failed" not "N in progress"; the wrapped update inside the+ # Surface the error to the UI + reconcile file counters so a failed run+ # reads "N failed" not "N in progress"; the wrapped update inside the# helper guarantees a status-update failure can't mask `e` or skip the- # re-raise below (see record_pg_orchestration_failure). The Celery branch- # is the original bare update, untouched.- if is_pg_transport(transport):- error_logger = None- try:- error_logger = WorkerWorkflowLogger.create_for_api_workflow(- execution_id=str(execution_id),- organization_id=str(schema_name),- pipeline_id=str(pipeline_id) if pipeline_id else None,- )- except Exception as log_error:- logger.warning(f"Failed to build UI logger: {log_error}")- WorkflowOrchestrationUtils.record_pg_orchestration_failure(- api_client=api_client,- execution_id=execution_id,- total_files=total_files,- error_message=error_message,- logger=logger,- workflow_logger=error_logger,- )- else:- # Update execution to ERROR status matching Django pattern- api_client.update_workflow_execution_status(- execution_id=execution_id,- status=ExecutionStatus.ERROR.value,- error_message=error_message,+ # re-raise below (see record_pg_orchestration_failure).+ error_logger = None+ try:+ error_logger = WorkerWorkflowLogger.create_for_api_workflow(+ execution_id=str(execution_id),+ organization_id=str(schema_name),+ pipeline_id=str(pipeline_id) if pipeline_id else None,)+ except Exception as log_error:+ logger.warning(f"Failed to build UI logger: {log_error}")+ WorkflowOrchestrationUtils.record_pg_orchestration_failure(+ api_client=api_client,+ execution_id=execution_id,+ total_files=total_files,+ error_message=error_message,+ logger=logger,+ workflow_logger=error_logger,+ )logger.error(f"Execution {execution_id} failed: {str(e)}", exc_info=True)raisediff --git a/workers/callback/tasks.py b/workers/callback/tasks.pyindex 94297ccd4..8259fff59 100644--- a/workers/callback/tasks.py+++ b/workers/callback/tasks.py@@ -365,7 +365,7 @@ def _update_execution_status_unified(aggregated_results: dict[str, Any],organization_id: str,error_message: str | None = None,- is_pg: bool = False,+ raise_on_failure: bool = True,) -> dict[str, Any]:"""Unified workflow execution status update for all callback types.@@ -379,6 +379,9 @@ def _update_execution_status_unified(aggregated_results: Aggregated file processing resultsorganization_id: Organization contexterror_message: Optional error message for failed executions+ raise_on_failure: Re-raise a failed status write (the default) so PG's+ at-least-once redelivery retries it. Pass ``False`` only where the+ caller is already handling a failure and a raise would mask it.Returns:Execution update result dictionary@@ -420,17 +423,16 @@ def _update_execution_status_unified(logger.error(f"Failed to update execution status for {execution_id}: {e}", exc_info=True)- # On the PG path this finalization write is the single terminalizing step.- # Swallowing a failure here strands the execution in EXECUTING forever: the- # consumer acks/deletes the message, the barrier row is already gone, and- # the reaper has no handle — every safety net is silently defeated. Re-raise- # so PG's at-least-once vt-redelivery retries the write and, if it keeps- # failing, poison-drops the batch to a terminal ERROR. A post-success- # redelivery is idempotency-guarded by _callback_already_ran (skips once- # COMPLETED; ERROR/STOPPED are intentionally re-run, so a failed-write- # ERROR execution simply retries the write). The Celery path keeps the- # legacy swallow (its chord retry covers it) so this is a PG-only change.- if is_pg:+ # This finalization write is the single terminalizing step. Swallowing a+ # failure here strands the execution in EXECUTING forever: the consumer+ # acks/deletes the message, the barrier row is already gone, and the reaper+ # has no handle — every safety net is silently defeated. Re-raise so PG's+ # at-least-once vt-redelivery retries the write and, if it keeps failing,+ # poison-drops the batch to a terminal ERROR. A post-success redelivery is+ # idempotency-guarded by _callback_already_ran (skips once COMPLETED;+ # ERROR/STOPPED are intentionally re-run, so a failed-write ERROR execution+ # simply retries the write).+ if raise_on_failure:raise# Return error result instead of re-raising to maintain callback flowreturn {@@ -1430,9 +1432,10 @@ def _process_batch_callback_core(# Initialize performance optimizations_initialize_performance_managers()- # PG at-least-once duplicate guard (see _callback_already_ran).- # Popped BEFORE parameter extraction so the marker never flows into the context.- is_pg = bool(kwargs.pop(PG_TRANSPORT_CALLBACK_KWARG, False))+ # Wire-compat only: PgBarrier stamps this marker on every callback it fires.+ # Nothing branches on it since UN-4078 (PG is the only transport), but it is+ # popped BEFORE parameter extraction so it never flows into the context.+ kwargs.pop(PG_TRANSPORT_CALLBACK_KWARG, None)# Extract and validate all parameters using single source of truthcontext = _extract_callback_parameters(task_instance, results, kwargs)@@ -1460,7 +1463,7 @@ def _process_batch_callback_core(# completed — don't re-fire webhooks / re-count billing. Reuses the# status from _extract_callback_parameters' fetch (no extra call).# (Returns through the outer finally, which closes the api_client.)- if is_pg and _callback_already_ran(context.execution_status):+ if _callback_already_ran(context.execution_status):logger.warning(f"PG callback: execution {context.execution_id} already COMPLETED "f"— a previous callback finalized it; skipping duplicate side "@@ -1486,7 +1489,6 @@ def _process_batch_callback_core(aggregated_results=aggregated_results,organization_id=context.organization_id,error_message=None,- is_pg=is_pg,)# Handle pipeline updates using unified function (non-API deployment)pipeline_result = _handle_pipeline_updates_unified(@@ -1566,6 +1568,9 @@ def _process_batch_callback_core({"error": str(e)[:500]},context.organization_id,error_message=str(e)[:500],+ # Already handling a failure: a raise here would replace+ # the original error with the status-write error.+ raise_on_failure=False,)logger.info(f"Marked execution {context.execution_id} as failed using unified function"@@ -1643,8 +1648,9 @@ def process_batch_callback_api(execution_id = kwargs.get("execution_id")pipeline_id = kwargs.get("pipeline_id")organization_id = kwargs.get("organization_id")- # PG at-least-once duplicate guard marker (see _callback_already_ran).- is_pg = bool(kwargs.pop(PG_TRANSPORT_CALLBACK_KWARG, False))+ # Wire-compat only: stamped by PgBarrier on every callback it fires. Nothing+ # branches on it since UN-4078; popped so it never flows into the context.+ kwargs.pop(PG_TRANSPORT_CALLBACK_KWARG, None)if not execution_id:raise ValueError("execution_id is required in kwargs")@@ -1677,7 +1683,7 @@ def process_batch_callback_api(# prior delivery already completed must skip its side effects. Reuses the# status just fetched — no extra round-trip. (Returns through the finally# below, which closes the api_client.)- if is_pg and _callback_already_ran(workflow_execution.get("status")):+ if _callback_already_ran(workflow_execution.get("status")):logger.warning(f"PG API callback: execution {execution_id} already COMPLETED — a "f"previous callback finalized it; skipping duplicate side effects "@@ -1746,7 +1752,6 @@ def process_batch_callback_api(final_status=execution_status,aggregated_results=aggregated_results,organization_id=organization_id,- is_pg=is_pg,)# Create minimal context for unified pipeline handlingdiff --git a/workers/file_processing/structure_tool_task.py b/workers/file_processing/structure_tool_task.pyindex 4b2f02bfc..95d745211 100644--- a/workers/file_processing/structure_tool_task.py+++ b/workers/file_processing/structure_tool_task.py@@ -1,17 +1,16 @@-"""Structure tool Celery task — Phase 3 of executor migration.+"""Structure tool task — Phase 3 of executor migration.-Replaces the Docker-container-based StructureTool.run() with a Celery-task that runs in the file_processing worker. Instead of PromptTool-HTTP calls to prompt-service, it uses ExecutionDispatcher to send-operations to the executor worker via Celery.+Replaces the Docker-container-based StructureTool.run() with a task that runs in+the file_processing worker. Instead of PromptTool HTTP calls to prompt-service,+it dispatches operations to the executor worker through the executor RPC.Before (Docker-based):File Processing Worker → WorkflowExecutionService → ToolSandbox→ Docker container → StructureTool.run() → PromptTool (HTTP) → prompt-service-After (Celery-based):+After:File Processing Worker → WorkerWorkflowExecutionService- → execute_structure_tool task → ExecutionDispatcher+ → execute_structure_tool task → executor RPC (PG queue)→ executor worker → LegacyExecutor"""@@ -276,10 +275,10 @@ def _merge_agentic_metrics(@worker_task(bind=True, name=str(TaskName.EXECUTE_STRUCTURE_TOOL))def execute_structure_tool(self, params: dict) -> dict:- """Execute structure tool as a Celery task.+ """Execute structure tool as a task.Replicates StructureTool.run() from tools/structure/src/main.py- but uses ExecutionDispatcher instead of PromptTool HTTP calls.+ but uses the executor RPC instead of PromptTool HTTP calls.Args:params: Dict with keys described in the Phase 3 plan.diff --git a/workers/file_processing/tasks.py b/workers/file_processing/tasks.pyindex 05b71ffaa..764953834 100644--- a/workers/file_processing/tasks.py+++ b/workers/file_processing/tasks.py@@ -54,7 +54,6 @@ from unstract.core.data_models import (FileHashData,PreCreatedFileData,WorkerFileData,- WorkflowTransport,)from unstract.core.worker_models import (ApiDeploymentResultStatus,@@ -239,10 +238,9 @@ class _TerminalExecutionSkip(Exception):resurrect the execution back to EXECUTING and re-run its files (double LLM /destination write). The batch is skipped instead.- Gated to the PG path (``is_pg``): a no-op on Celery, so the existing Celery- flow is behaviorally unchanged. It's PG-specific because the **reaper** that- marks an in-flight execution terminal (and tears its barrier down) is a- PG-only mechanism — the Celery chord has no equivalent.+ The resurrection this prevents exists because the **reaper** can mark an+ in-flight execution terminal (and tear its barrier down) while a batch for it+ is still in flight or waiting to be redelivered.This is a validate-first *narrowing*, not a transactional guarantee: thestatus read and the later ``EXECUTING`` write are not atomic, so a reaper@@ -260,20 +258,16 @@ class _TerminalExecutionSkip(Exception):def _raise_if_execution_terminal(- workflow_execution: dict[str, Any], execution_id: str, *, is_pg: bool+ workflow_execution: dict[str, Any], execution_id: str) -> None:- """PG-path-only validate-first guard: refuse to (re)process a batch whose- execution is already terminal. See :class:`_TerminalExecutionSkip`.-- No-op on the Celery path (``is_pg=False``) — the resurrection this prevents- is PG-specific, so gating leaves the Celery flow behaviorally unchanged. A- missing or unrecognized status is treated as non-terminal (fail open —- proceed as normal), but both are logged on the PG path so a degraded- execution-fetch response or a new/renamed status isn't silently masked (an- unrecognized terminal-ish state would otherwise let a stale batch through).+ """Validate-first guard: refuse to (re)process a batch whose execution is+ already terminal. See :class:`_TerminalExecutionSkip`.++ A missing or unrecognized status is treated as non-terminal (fail open —+ proceed as normal), but both are logged so a degraded execution-fetch+ response or a new/renamed status isn't silently masked (an unrecognized+ terminal-ish state would otherwise let a stale batch through)."""- if not is_pg:- returnstatus = workflow_execution.get("status")if not status:# Fail open (proceed), but surface it — a missing status is an@@ -319,17 +313,13 @@ def _terminal_skip_result(batch_data: FileBatchData) -> dict[str, Any]:def _run_batch_stages(- file_batch_data: dict[str, Any], celery_task_id: str, *, is_pg: bool+ file_batch_data: dict[str, Any], celery_task_id: str) -> dict[str, Any]:"""The actual batch work (validate → setup → pre-create → process → compile).- Transport-agnostic: identical on the Celery chord path and the PG- fire-and-forget path. The task instance isn't needed here — its only use- (deriving ``celery_task_id``) happens in the caller. Returns the- JSON-serialisable batch result.-- ``is_pg`` (keyword-only, no default so a PG call site can't silently forget- it) forwards to the terminal guard; see :class:`_TerminalExecutionSkip`.+ The task instance isn't needed here — its only use (deriving+ ``celery_task_id``) happens in the caller. Returns the JSON-serialisable+ batch result."""# Step 1: Validate and parse input databatch_data = _validate_and_parse_batch_data(file_batch_data)@@ -338,7 +328,7 @@ def _run_batch_stages(# stale/redelivered batch for an already-terminal execution — see# :class:`_TerminalExecutionSkip`.try:- context = _setup_execution_context(batch_data, celery_task_id, is_pg=is_pg)+ context = _setup_execution_context(batch_data, celery_task_id)except _TerminalExecutionSkip as skip:logger.warning(f"[exec:{skip.execution_id}] Skipping batch of "@@ -374,11 +364,11 @@ def _process_file_batch_core(Args:task_instance: The Celery task instance (self)file_batch_data: Dictionary that will be converted to FileBatchData dataclass- barrier_context: Present only on the 9e PG fire-and-forget path — carries- ``execution_id`` / ``batch_index`` / ``callback_descriptor`` so the- batch claims its slot and runs the barrier decrement in-body (a- PG-consumed task fires no Celery ``.link``). ``None`` on the Celery- chord path, where the chord ``.link`` drives the decrement instead.+ barrier_context: Carries ``execution_id`` / ``batch_index`` /+ ``callback_descriptor`` so the batch claims its slot and runs the+ barrier decrement in-body. Every fan-out injects it+ (``PgBarrier._dispatch_header_pg``), so ``None`` means a malformed+ payload — see the guard below, which logs and still runs the batch.Returns:Dictionary with successful_files and failed_files counts@@ -388,17 +378,27 @@ def _process_file_batch_core()if barrier_context is None:- # Celery chord path — the chord's .link runs the decrement after this.- # is_pg=False disables the terminal guard → Celery flow unchanged.- return _run_batch_stages(file_batch_data, celery_task_id, is_pg=False)-- # PG fire-and-forget path — claim the batch (idempotent on redelivery), run- # the stages, then decrement the barrier in-body / self-chain the callback.- # is_pg=True enables the terminal guard (skip a stale/redelivered batch for a- # reaper-recovered execution).+ # Every fan-out injects ``_barrier_context`` (PgBarrier._dispatch_header_pg),+ # so this means a malformed or hand-crafted payload. Run the batch — the+ # work is still well-defined — but say so loudly: with no barrier to+ # decrement, nothing will fire the aggregating callback and the execution+ # will sit until the reaper reclaims it.+ file_data = file_batch_data.get("file_data") or {}+ logger.error(+ f"[exec:{file_data.get('execution_id')}] "+ f"[workflow:{file_data.get('workflow_id')}] process_file_batch received "+ f"no _barrier_context for a batch of "+ f"{len(file_batch_data.get('files') or [])} file(s); the batch will run "+ "but no barrier will be decremented and the aggregating callback will "+ "never fire. This indicates a malformed payload."+ )+ return _run_batch_stages(file_batch_data, celery_task_id)++ # Claim the batch (idempotent on redelivery), run the stages, then decrement+ # the barrier in-body / self-chain the callback.return run_batch_with_barrier(barrier_context,- lambda: _run_batch_stages(file_batch_data, celery_task_id, is_pg=True),+ lambda: _run_batch_stages(file_batch_data, celery_task_id),)@@ -425,10 +425,9 @@ def process_file_batch(Args:file_batch_data: Dictionary that will be converted to FileBatchData dataclass- _barrier_context: Injected only when this task is dispatched onto the PG- queue (9e fire-and-forget path) by ``PgBarrier`` — carries the barrier- coordination context (``execution_id`` / ``batch_index`` /- ``callback_descriptor``). Absent on the Celery chord path.+ _barrier_context: Injected by ``PgBarrier`` on every fan-out — carries the+ barrier coordination context (``execution_id`` / ``batch_index`` /+ ``callback_descriptor``). Absent only on a malformed payload.Returns:Dictionary with successful_files and failed_files counts@@ -470,24 +469,21 @@ def _validate_and_parse_batch_data(file_batch_data: dict[str, Any]) -> FileBatchdef _setup_execution_context(- batch_data: FileBatchData, celery_task_id: str, *, is_pg: bool+ batch_data: FileBatchData, celery_task_id: str) -> WorkflowContextData:"""Setup execution context with validation and API client initialization.Args:batch_data: Validated batch data- celery_task_id: Celery task ID for tracking- is_pg: Whether this batch runs on the PG (at-least-once) transport.- Keyword-only, no default so a PG call site can't silently forget it.- Enables the terminal guard; a no-op on Celery.+ celery_task_id: Task ID for trackingReturns:WorkflowContextData containing type-safe execution contextRaises:ValueError: If required context fields are missing- _TerminalExecutionSkip: On the PG path, if the execution is already- terminal (a stale/redelivered batch after reaper-recovery).+ _TerminalExecutionSkip: If the execution is already terminal (a+ stale/redelivered batch after reaper-recovery)."""# Extract context using dataclassfile_data = batch_data.file_data@@ -527,10 +523,10 @@ def _setup_execution_context(execution_context = execution_response.dataworkflow_execution = execution_context.get("execution", {})- # Validate-first terminal guard (PG only) — skip a stale/redelivered batch+ # Validate-first terminal guard — skip a stale/redelivered batch# for an already-terminal execution, before we touch status or process# anything. See :class:`_TerminalExecutionSkip`.- _raise_if_execution_terminal(workflow_execution, execution_id, is_pg=is_pg)+ _raise_if_execution_terminal(workflow_execution, execution_id)# Set LOG_EVENTS_ID in StateStore for WebSocket messaging (critical for UI logs)# This enables the WorkerWorkflowLogger to send logs to the UI via WebSocket@@ -599,18 +595,12 @@ def _setup_execution_context(f"File history from fallback access for workflow {workflow_id}: use_file_history = {use_file_history}")- # Create type-safe workflow context. Set the transport authoritatively from- # is_pg (the batch-level fact) — WorkerFileData carries none — so the- # destination layer can apply PG-only guards.context_data = WorkflowContextData(workflow_id=workflow_id,workflow_name=workflow_name,workflow_type=workflow_type,execution_id=execution_id,organization_context=org_context,- transport=(- WorkflowTransport.PG_QUEUE.value if is_pg else WorkflowTransport.CELERY.value- ),files={f"file_{i}": file for i, file in enumerate(files)}, # Convert list to dict format@@ -875,7 +865,6 @@ def _process_individual_files(context: WorkflowContextData) -> WorkflowContextDaworkflow_file_execution_id=workflow_file_execution_id, # Pass pre-created IDworkflow_file_execution_object=workflow_file_execution_object, # Pass pre-created objectworkflow_logger=workflow_logger, # Pass workflow logger for UI logging- transport=context.transport, # Drives the PG-only destination guard)# Handle file processing result@@ -1645,7 +1634,6 @@ def _process_file(workflow_file_execution_id: str = None,workflow_file_execution_object: Any = None,workflow_logger: Any = None,- transport: str | None = None,) -> dict[str, Any]:"""Process a single file matching Django backend _process_file pattern.@@ -1659,8 +1647,6 @@ def _process_file(file_hash: FileHashData instance with type-safe accessapi_client: Internal API clientworkflow_execution: Workflow execution context- transport: Execution transport (celery | pg_queue); drives PG-only- destination guards downstream.Returns:File execution result@@ -1676,7 +1662,6 @@ def _process_file(workflow_file_execution_id=workflow_file_execution_id,workflow_file_execution_object=workflow_file_execution_object,workflow_logger=workflow_logger,- transport=transport,)@@ -2242,56 +2227,36 @@ def process_file_batch_resilient(def process_file_batch_django_compat(self, file_batch_data: dict[str, Any]) -> dict[str, Any]:- """Backward compatibility wrapper for Django backend task name.+ """Backward compatibility wrapper for the old Django backend task name.- This allows new workers to handle tasks sent from the old Django backend- during the transition period when both systems are running.+ Registered so a task sent under the pre-workers name is still recognised.+ Its only producer was the in-backend Celery chord, which UN-4078 removed+ along with the rest of the Celery transport.++ It carries no ``_barrier_context``, so the batch would run with no claim+ (no ``pg_batch_dedup`` marker, so a redelivery re-runs the whole batch: the+ LLM spend twice, and a duplicate destination write when+ ``use_file_history=False``) and with nothing to fire the aggregating+ callback. Refuse instead — the same choice ``step_execution`` makes on the+ backend — so the failure is visible rather than a silent double-spend.Args:- file_batch_data: File batch data from Django backend+ file_batch_data: File batch data from the old Django backend- Returns:- Same result as process_file_batch+ Raises:+ RuntimeError: always."""- logger.info(- "Processing file batch via Django compatibility task name: "- "workflow_manager.workflow_v2.file_execution_tasks.process_file_batch"+ file_data = file_batch_data.get("file_data") or {}+ raise RuntimeError(+ f"[exec:{file_data.get('execution_id')}] "+ f"[workflow:{file_data.get('workflow_id')}] the Django-compat task name "+ "workflow_manager.workflow_v2.file_execution_tasks.process_file_batch is no "+ "longer executable: it carries no barrier context, so the batch would run "+ "unclaimed and no aggregating callback would fire. Its only producer was the "+ "Celery chord removed in UN-4078. Dispatch process_file_batch through "+ "PgBarrier instead.")- # Django compatibility: Calculate and apply manual review requirements- # This replicates the MRQ logic that was originally in Django backend- try:- # Extract organization_id from Django backend data structure- # Django sends: {files: [...], file_data: {organization_id: "...", ...}}- file_data = file_batch_data.get("file_data", {})- organization_id = file_data.get("organization_id")-- if not organization_id:- logger.warning(- "Django compatibility: No organization_id found in file_data, skipping MRQ calculation"- )- else:- # Create organization-scoped API client- api_client = create_api_client(organization_id)-- # Calculate manual review requirements- mrq_flags = _calculate_manual_review_requirements(file_batch_data, api_client)-- # Enhance batch data with MRQ flags- _enhance_batch_with_mrq_flags(file_batch_data, mrq_flags)-- logger.info(- f"Django compatibility: Applied manual review flags to file batch for org {organization_id}"- )-- except Exception as e:- logger.warning(f"Django compatibility: Failed to calculate MRQ flags: {e}")- raise- # Continue processing without MRQ flags rather than failing-- # Delegate to the core implementation (same as main task)- return _process_file_batch_core(self, file_batch_data)-# Helper functions for refactored _handle_file_processing_resultdiff --git a/workers/general/tasks.py b/workers/general/tasks.pyindex 7200a6589..861ddc7dd 100644--- a/workers/general/tasks.py+++ b/workers/general/tasks.py@@ -56,13 +56,10 @@ from worker import app, config# Import shared data models for type safetyfrom unstract.core.data_models import (- DEFAULT_WORKFLOW_TRANSPORT,ExecutionStatus,FileBatchData,FileHashData,WorkerFileData,- is_pg_transport,- normalize_transport,)# Import common workflow utilities@@ -143,16 +140,15 @@ def _log_batch_creation_statistics(def _should_skip_duplicate_orchestration(- execution_id: str, organization_id: str, transport: str, retries: int = 0+ execution_id: str, organization_id: str, retries: int = 0) -> bool:- """PG-only: claim the execution's orchestration slot; ``True`` iff THIS- delivery lost the claim (a duplicate / redelivered orchestration) and must- no-op.+ """Claim the execution's orchestration slot; ``True`` iff THIS delivery lost+ the claim (a duplicate / redelivered orchestration) and must no-op.``organization_id`` (the org schema_name) is stamped onto the claim row so thereaper can call the org-scoped API when it recovers or GCs an orphan claim.- On the PG queue the orchestration task can be redelivered — it runs longer+ The orchestration task can be redelivered — it runs longerthan the consumer's visibility timeout and a sibling replica re-claims themessage — and re-running it re-arms the barrier (resets ``remaining`` andwipes the dedup markers mid-flight) and dispatches a second set of batch@@ -165,14 +161,7 @@ def _should_skip_duplicate_orchestration(a prior attempt claimed and did not release it (a hard crash after the claim,before the failure-path release) — a suppressed retry, logged at ERROR with anerrorId so it's distinguishable from a normal duplicate.-- No-op on Celery (returns ``False`` — always proceeds): Celery has no- at-least-once redelivery, and the router owns terminal status there, so there- is no double-orchestration to guard. Gated exactly like the terminal guard so- the Celery path is behaviorally unchanged."""- if not is_pg_transport(transport):- return Falseif try_claim_orchestration(execution_id, organization_id):return False # first delivery — this replica owns the orchestrationif retries > 0:@@ -213,7 +202,6 @@ def async_execute_bin_general(pipeline_id: str | None = None,log_events_id: str | None = None,use_file_history: bool = False,- transport: str = DEFAULT_WORKFLOW_TRANSPORT,**kwargs: dict[str, Any],) -> dict[str, Any]:"""Lightweight general workflow execution task.@@ -248,27 +236,27 @@ def async_execute_bin_general(f"Starting general workflow execution for workflow {workflow_id}, execution {execution_id}")- # Tracks whether THIS delivery won the PG orchestration claim, so the- # error path releases the claim only when we actually acquired it (not, for+ # Tracks whether THIS delivery won the orchestration claim, so the error+ # path releases the claim only when we actually acquired it (not, for# instance, when the claim's own INSERT raised). Set below only after the- # gate returns "proceed" on the PG path.+ # gate returns "proceed".claimed_orchestration = Falsetry:- # PG-only idempotency gate (claim-before-work): a duplicate /- # redelivered orchestration no-ops here BEFORE any setup, barrier arm,- # or batch dispatch — so exactly one delivery orchestrates the- # execution even with replicas > 1. No-op on Celery.+ # Idempotency gate (claim-before-work): a duplicate / redelivered+ # orchestration no-ops here BEFORE any setup, barrier arm, or batch+ # dispatch — so exactly one delivery orchestrates the execution even+ # with replicas > 1.if _should_skip_duplicate_orchestration(- execution_id, schema_name, transport, self.request.retries+ execution_id, schema_name, self.request.retries):return {"status": "skipped_duplicate_orchestration","execution_id": execution_id,"workflow_id": workflow_id,}- # Gate returned "proceed": on PG that means this delivery just won the- # claim (Celery never claims), so the error path owns releasing it.- claimed_orchestration = is_pg_transport(transport)+ # Gate returned "proceed": this delivery just won the claim, so the+ # error path owns releasing it.+ claimed_orchestration = True# Initialize execution context with shared utilityconfig, api_client = WorkerExecutionContext.setup_execution_context(@@ -312,16 +300,11 @@ def async_execute_bin_general(# to test) a redelivery could reach here, win a fresh claim, and re-arm +# re-dispatch a finished execution — duplicate (costly) destination# writes when use_file_history=False. Skip if already terminal, keeping- # the freshly-won claim so it re-establishes the tombstone. PG only:- # Celery has no redelivery, so the orchestrator only ever runs on a- # non-terminal execution — the check is a no-op there.- if (- is_pg_transport(transport)- and current_status in ExecutionStatus.terminal_values()- ):+ # the freshly-won claim so it re-establishes the tombstone.+ if current_status in ExecutionStatus.terminal_values():logger.warning(f"Execution {execution_id} already terminal ({current_status}) "- f"at orchestration entry — skipping re-orchestration (a PG "+ f"at orchestration entry — skipping re-orchestration (a "f"redelivery of a finished execution).")return {@@ -369,7 +352,6 @@ def async_execute_bin_general(use_file_history,scheduled,schema_name,- transport=transport,**kwargs,)@@ -605,7 +587,6 @@ def _execute_general_workflow(use_file_history: bool,scheduled: bool,schema_name: str,- transport: str = DEFAULT_WORKFLOW_TRANSPORT,**kwargs: dict[str, Any],) -> dict[str, Any]:"""Execute general workflow specific logic for ETL/TASK workflows.@@ -628,12 +609,6 @@ def _execute_general_workflow("""start_time = time.time()- # Fail-closed coercion, for parity with the api/scheduler workers (which run- # normalize_transport at their entry): a typo'd transport degrades to Celery- # with a warning rather than silently routing onto an unknown substrate. The- # coerced value feeds both WorkflowContextData and the fan-out below.- transport = normalize_transport(transport, logger=logger)-logger.info("Executing general workflow logic for ETL/TASK workflow")try:@@ -672,7 +647,6 @@ def _execute_general_workflow("execution_mode": execution_mode,},is_scheduled=scheduled,- transport=transport,)logger.info(@@ -856,7 +830,6 @@ def _execute_general_workflow(execution_mode=execution_mode,use_file_history=use_file_history,organization_id=api_client.organization_id,- transport=transport,**kwargs,)@@ -868,28 +841,17 @@ def _execute_general_workflow(except Exception as e:logger.error(f"Workflow execution failed: {e}")- # On PG, surface the error to the UI + reconcile file counters so a- # failed run reads "N failed" not "N in progress" (see- # WorkflowOrchestrationUtils.record_pg_orchestration_failure). The- # Celery branch below is the original status update, untouched.- if is_pg_transport(transport):- WorkflowOrchestrationUtils.record_pg_orchestration_failure(- api_client=api_client,- execution_id=execution_id,- total_files=total_files,- error_message=str(e),- logger=logger,- workflow_logger=workflow_logger,- )- else:- try:- api_client.update_workflow_execution_status(- execution_id=execution_id,- status=ExecutionStatus.ERROR.value,- error_message=str(e),- )- except Exception as status_error:- logger.warning(f"Failed to update error status: {status_error}")+ # Surface the error to the UI + reconcile file counters so a failed+ # run reads "N failed" not "N in progress" (see+ # WorkflowOrchestrationUtils.record_pg_orchestration_failure).+ WorkflowOrchestrationUtils.record_pg_orchestration_failure(+ api_client=api_client,+ execution_id=execution_id,+ total_files=total_files,+ error_message=str(e),+ logger=logger,+ workflow_logger=workflow_logger,+ )orchestration_result = WorkerTaskResponse.error_response(execution_id=execution_id,@@ -925,7 +887,6 @@ def _orchestrate_file_processing_general(execution_mode: tuple | None,use_file_history: bool,organization_id: str,- transport: str = DEFAULT_WORKFLOW_TRANSPORT,**kwargs: dict[str, Any],) -> dict[str, Any]:"""Orchestrate file processing for general workflows using the same pattern as API worker.@@ -1119,7 +1080,6 @@ def _orchestrate_file_processing_general(org_id=organization_id,workload_type=WorkloadType.NON_API,),- transport=transport,)if not result:diff --git a/workers/queue_backend/__init__.py b/workers/queue_backend/__init__.pyindex e148708a6..febd02104 100644--- a/workers/queue_backend/__init__.py+++ b/workers/queue_backend/__init__.py@@ -1,128 +1,37 @@"""Queue-backend seam.-Single place where the substrate choice (Celery today; PG Queue later)-lives. Both entry points are transparent passthroughs to Celery today.--The ``Barrier`` substrate is runtime-selectable via the-``WORKER_BARRIER_BACKEND`` env var:--- ``chord`` (default) — ``CeleryChordBarrier``, the thin wrapper around- ``celery.chord(header)(body)``. Byte-identical wire to the- pre-abstraction code path; current production behaviour.-- ``redis`` — ``RedisDecrBarrier``, the labs-design distributed-counter- pattern (``DECR remaining:{exec_id}`` + per-task ``RPUSH``-based- result aggregation). Replaces only the chord aggregation primitive;- dispatch / retries / monitoring still ride on Celery.--**Default-safety posture.** The env defaults to ``chord``, so a merge-into ``main`` is zero-impact: a downstream env (``unstract-cloud``,-self-hosted, etc.) that does not set ``WORKER_BARRIER_BACKEND`` runs-the same substrate it ran yesterday. The new ``RedisDecrBarrier`` code-is dormant until an operator explicitly flips the flag.--Unknown values raise — operators get a loud error at module import-time rather than silently falling back, which would mask a typo'd-flag in a production env.--**Queue-transport routing (separate axis).** ``dispatch()`` consults-:func:`~queue_backend.routing.select_backend`, which returns PG-unconditionally since UN-4046. This is orthogonal to the barrier choice-above: the barrier is *how a batch's fan-in fires the callback*; the-transport is *how messages travel*. It is emphatically NOT inert — every-dispatch rides PG and needs its consumer running, or the message is-durably enqueued and never executed, with no error at the producer.-(It used to read a ``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list that was-set nowhere, so everything defaulted to Celery; both are gone.)+Single place where the queue substrate lives. Postgres is the only transport and+:class:`~queue_backend.pg_barrier.PgBarrier` the only fan-in barrier.++**History.** Both were once selectable. ``dispatch()`` read a+``WORKER_PG_QUEUE_ENABLED_TASKS`` allow-list (set nowhere, so everything+defaulted to Celery — removed in UN-4046), and the barrier substrate was chosen+at import time by ``WORKER_BARRIER_BACKEND`` from ``chord`` /+``redis`` / ``pg``. The Celery and Redis barriers, the env var and the+``get_barrier()`` factory went with the Celery transport in UN-4078.++**Every dispatch rides PG and needs its consumer running**, or the message is+durably enqueued and never executed, with no error at the producer. The+deployment chart (``validate-pg-worker-fleet.yaml``, in the closed-source+``unstract-cloud`` repo — it is not in this tree) refuses a partial fleet at+render time for exactly this reason."""-import os-from enum import StrEnum--from .barrier import Barrier, BarrierHandle, CeleryChordBarrier+from .barrier import Barrier, BarrierHandlefrom .decorator import worker_taskfrom .dispatch import dispatchfrom .fairness import FairnessKey-from .pg_barrier import (- PgBarrier,- barrier_pg_abort,- barrier_pg_decr_and_check,-)-from .redis_barrier import (- RedisDecrBarrier,- barrier_abort,- barrier_decr_and_check,-)+from .pg_barrier import PgBarrier, barrier_pg_abortfrom .routing import QueueBackend, select_backend__all__ = ["Barrier",- "BarrierBackend","BarrierHandle",- "CeleryChordBarrier","FairnessKey","PgBarrier","QueueBackend",- "RedisDecrBarrier",- "barrier_abort",- "barrier_decr_and_check","barrier_pg_abort",- "barrier_pg_decr_and_check","dispatch",- "get_barrier","select_backend","worker_task",]---class BarrierBackend(StrEnum):- """Enumeration of ``Barrier`` substrate choices.-- Used as the canonical reference for the ``WORKER_BARRIER_BACKEND``- env value, the factory's dispatch table, and tests that flip the- env. Subclassing ``StrEnum`` (Python 3.11+) means ``BarrierBackend.CHORD- == "chord"`` at runtime, so the enum members can stand in anywhere- a string is expected — including ``os.environ`` reads.- """-- CHORD = "chord"- REDIS = "redis"- PG = "pg"---def get_barrier() -> Barrier:- """Return the ``Barrier`` implementation selected by env.-- Reads ``WORKER_BARRIER_BACKEND`` at call time so test harnesses- that ``monkeypatch.setenv`` can flip the substrate per-test- without a module reload. Production call sites construct the- barrier once per process (module-level singleton in- ``orchestration_utils.py``) so the env read happens at worker- startup — flag flips require pod restart, same posture as every- other ``WORKER_*`` env in the codebase.-- Default ``BarrierBackend.CHORD`` mirrors the current production- substrate. An unknown value raises ``ValueError`` so a typo'd- ``redis`` → ``rediz`` doesn't silently fall back to chord (which- would invalidate the canary).- """- raw = os.getenv("WORKER_BARRIER_BACKEND", BarrierBackend.CHORD.value)- try:- backend = BarrierBackend(raw.strip().lower())- except ValueError:- valid = tuple(b.value for b in BarrierBackend)- raise ValueError(- f"WORKER_BARRIER_BACKEND={raw!r} is not a recognised "- f"barrier backend. Valid values: {valid}. Unset the env var "- f"to default to {BarrierBackend.CHORD.value!r}."- ) from None-- if backend is BarrierBackend.CHORD:- return CeleryChordBarrier()- if backend is BarrierBackend.REDIS:- return RedisDecrBarrier()- if backend is BarrierBackend.PG:- return PgBarrier()- # Unreachable — StrEnum constructor would have raised above for- # anything not in the enum. Defensive raise so the type checker- # sees an exhaustive match.- raise AssertionError(f"Unhandled BarrierBackend: {backend!r}")diff --git a/workers/queue_backend/barrier.py b/workers/queue_backend/barrier.pyindex 4695765de..0bc773c22 100644--- a/workers/queue_backend/barrier.py+++ b/workers/queue_backend/barrier.py@@ -1,46 +1,30 @@-"""Distributed barrier — abstraction over the chord-completion pattern.--The two production chord call sites (``shared/workflow/execution/-orchestration_utils.py`` and ``api-deployment/tasks.py``) need to wait-for a fan-out of header tasks to all complete, then fire a callback-with the aggregated results. Today that's implemented by Celery's-``chord(header)(body)`` primitive, which is the highest-risk Celery-construct at our scale — silent task drops at ~130K-task scale are-documented in the PG Queue decision journey.--This module provides the ``Barrier`` protocol and a ``CeleryChordBarrier``-implementation that wraps the existing ``chord(...)`` call 1:1. Lifting-both call sites to ``Barrier`` gives Phase 6b a place to land an-alternative implementation (e.g. ``RedisDecrBarrier`` using the labs-``DECR remaining:{exec_id}`` pattern, or — later in Phase 8 — a-``PgBarrier`` using SKIP LOCKED) without touching the call sites a-second time.--**Phase 6a (this PR): wrapper only — behaviour-preserving uplift.**-``CeleryChordBarrier.enqueue(...)`` produces byte-identical wire output-to the direct ``chord(...)`` calls it replaces *when ``fairness=None``*.-When a ``FairnessKey`` is passed (which both production call sites-now do), each header task and the callback additionally carry an-``x-fairness-key`` AMQP header. That's an additive wire-level change —-consumers ignore unknown headers — same posture as Phase 5.1's-``dispatch()`` plumbing. Mixed-version rolling deploys are safe in-both directions: old workers ignore the new header; new workers-handle messages without the header identically to today.--Phase 6b (separate PR) will add ``RedisDecrBarrier`` as a second-implementation, gated by ``WORKER_BARRIER_BACKEND`` env flag (default-``chord``). Phase 6c is the rollout.+"""Distributed barrier — the fan-out-then-callback seam.++The two production fan-out call sites (``shared/workflow/execution/+orchestration_utils.py`` and ``api-deployment/tasks.py``) need to wait for a+fan-out of header tasks to all complete, then fire a callback with the+aggregated results.++This module holds the ``Barrier`` protocol and the types that cross the+producer → queue → consumer boundary. The single implementation is+:class:`~queue_backend.pg_barrier.PgBarrier`.++**History.** This seam existed to let the substrate be swapped without touching+the call sites twice. It carried three implementations: ``CeleryChordBarrier``+(``celery.chord``, the original), ``RedisDecrBarrier`` (a ``DECR`` counter), and+``PgBarrier``. Selection was runtime, via ``WORKER_BARRIER_BACKEND``. The first+two, the env var, and the ``get_barrier()`` factory were deleted in UN-4078 once+PG became the only transport. Nothing annotates against the protocol any more —+both fan-out sites construct :class:`PgBarrier` concretely — so it survives only+as documentation of the seam's shape, alongside the shared TypedDicts below,+which ARE load-bearing. Deleting it is a separate post-decommission cleanup."""from __future__ import annotationsimport loggingimport os-from typing import TYPE_CHECKING, Any, NotRequired, Protocol, TypedDict--from celery import chord--from unstract.core.data_models import DEFAULT_WORKFLOW_TRANSPORT+from typing import TYPE_CHECKING, Any, Literal, Protocol, TypedDictfrom .fairness import FairnessKeyfrom .handle import BarrierHandle@@ -82,9 +66,9 @@ def _positive_int_env(var: str, default: int, human_default: str) -> int:def barrier_ttl_seconds() -> int:"""Barrier TTL from ``WORKER_BARRIER_KEY_TTL_SECONDS`` (default 6h).- Read at call time so tests can flip it. Invalid / non-positive values raise,- matching ``get_barrier()``'s loud-on-misconfig posture — a TTL shorter than- execution wall-clock would tear barriers down early (spurious behaviour).+ Read at call time so tests can flip it. Invalid / non-positive values raise+ rather than falling back — a TTL shorter than execution wall-clock would tear+ barriers down early (spurious behaviour), so a garbled value must fail loud."""return _positive_int_env("WORKER_BARRIER_KEY_TTL_SECONDS",@@ -126,23 +110,24 @@ def barrier_stuck_timeout_seconds() -> int:class CallbackDescriptor(TypedDict):"""Serialisable aggregating-callback spec baked into a barrier link signature.- Crosses a Celery serialisation boundary (producer → broker → worker), so the- four-key contract is typed to catch a typo/rename before it surfaces as a- remote ``KeyError`` mid-aggregation. Shared by both the Redis and PG- backends. ``fairness_headers`` is ``None`` when the producer passed no key.+ Crosses a serialisation boundary (producer → queue → worker), so the contract+ is typed to catch a typo/rename before it surfaces as a remote ``KeyError``+ mid-aggregation. ``fairness_headers`` is ``None`` when the producer passed no+ key."""task_name: strkwargs: dict[str, Any]queue: strfairness_headers: dict[str, Any] | None- # 9e: the WorkflowTransport the aggregating callback is fired on when the- # barrier completes. Absent / ``None`` → legacy Celery dispatch- # (``current_app.apply_async`` — the ``.link`` path). ``"pg_queue"`` → the- # fire-and-forget PG path self-chains the callback via dispatch onto PG.- # Named ``transport`` (a WorkflowTransport value, e.g. ``"pg_queue"``) — NOT- # ``backend``, to avoid confusion with ``QueueBackend`` (``"pg"``).- transport: NotRequired[str | None]+ # Rolling-deploy shim, see LEGACY_TRANSPORT_KEY in unstract.core.data_models.+ # REQUIRED for the shim release, not optional: a pre-UN-4078 consumer reads+ # ``callback_descriptor.get("transport")`` and, finding nothing, fires the+ # aggregating callback through Celery onto a broker with no consumers —+ # stranding the execution. ``Literal`` rejects "celery", the one value that+ # must never be written here, and makes the follow-up removal a type error at+ # the write site rather than a grep.+ transport: Literal["pg_queue"]class BarrierContext(TypedDict):@@ -184,8 +169,8 @@ class Barrier(Protocol):Semantically: "enqueue these header tasks in parallel; when allcomplete, fire this callback with the list of header results."- The concrete substrate (Celery chord today, Redis DECR or PG- SKIP LOCKED later) is hidden behind this protocol.+ :class:`~queue_backend.pg_barrier.PgBarrier` (PG ``SKIP LOCKED``) is the sole+ implementation since UN-4078."""def enqueue(@@ -197,15 +182,9 @@ class Barrier(Protocol):callback_queue: str,app_instance: Any,fairness: FairnessKey | None = None,- transport: str = DEFAULT_WORKFLOW_TRANSPORT,) -> BarrierHandle | None:"""Enqueue ``header_tasks`` and a single callback to fire on completion.- ``transport`` is the per-execution transport (9e). Only ``PgBarrier``- acts on it (``pg_queue`` → fire-and-forget PG fan-out instead of Celery- ``.link``); the Celery/Redis substrates accept it for Protocol parity- and ignore it (they are only ever reached on the ``celery`` transport).-``None`` is the **sole signal** that no work was enqueued — itis returned exclusively when ``header_tasks`` is empty. Anysubstrate-level failure (broker outage, serialisation error,@@ -221,117 +200,9 @@ class Barrier(Protocol):callback — same wire shape as ``dispatch()`` uses for baresites. Pass ``None`` for non-workflow-execution callers.- ``app_instance`` is a Celery app today (the Protocol leaks- this concrete dependency for the same reason the- ``CeleryChordBarrier`` does — Phase 6b's RedisDecrBarrier will- narrow this to a smaller Protocol). The argument is the- ``app`` from the producer's task context and must expose- ``.signature(name, kwargs=..., queue=..., headers=...)``.+ ``app_instance`` is accepted for backward compatibility with the two+ call sites and is unused by :class:`PgBarrier` — the header tasks are+ dispatched onto the PG queue by name, not through a Celery app. It goes+ with the protocol in the post-decommission cleanup."""...---class CeleryChordBarrier:- """Thin wrapper around ``celery.chord``.-- Behaviour-preserving: the ``chord(header_tasks)(callback_signature)``- call shape is the same as the inline ``chord(...)`` it replaces.- Wire output is byte-identical to the pre-Barrier path when- ``fairness=None``; when a ``FairnessKey`` is passed the only- addition is the ``x-fairness-key`` AMQP header on each header- task and the callback (additive wire-level change — consumers- that don't read the header are unaffected, same backward-compat- posture as Phase 5.1's ``dispatch()`` fairness plumbing).-- Future ``RedisDecrBarrier`` and ``PgBarrier`` implementations will- live alongside this class and be selected by the- ``WORKER_BARRIER_BACKEND`` env flag (Phase 6b).- """-- def enqueue(- self,- header_tasks: list[Signature],- *,- callback_task_name: str,- callback_kwargs: dict[str, Any],- callback_queue: str,- app_instance: Any,- fairness: FairnessKey | None = None,- transport: str = DEFAULT_WORKFLOW_TRANSPORT,- ) -> BarrierHandle | None:- """See :class:`Barrier.enqueue`."""- del transport # Celery chord is only reached on the celery transport.- # Empty-header guard goes FIRST so a zero-task run skips- # signature construction / fairness serialisation entirely.- # Any failure in those paths is now constrained to the- # non-empty branch where the work is actually needed.- if not header_tasks:- execution_id = callback_kwargs.get("execution_id")- pipeline_id = callback_kwargs.get("pipeline_id")- logger.info(- f"[exec:{execution_id}] [pipeline:{pipeline_id}] "- "Zero header tasks detected — skipping barrier enqueue "- "(parent should handle pipeline status updates directly)"- )- return None-- try:- # Use ``fairness.as_header()`` — the single wire-shape- # encoder, also used by ``dispatch.py``. Constructing the- # header dict inline here would risk silent divergence- # from the bare-dispatch path.- fairness_headers = fairness.as_header() if fairness else None-- # Build the callback-signature kwargs explicitly. When- # ``fairness=None`` the resulting call shape is- # byte-identical to the pre-Barrier ``app.signature(name,- # kwargs=..., queue=...)`` (no spurious ``headers=None``).- signature_kwargs: dict[str, Any] = {- "kwargs": callback_kwargs,- "queue": callback_queue,- }- if fairness_headers:- signature_kwargs["headers"] = fairness_headers- callback_signature = app_instance.signature(- callback_task_name, **signature_kwargs- )-- # Stamp fairness onto each header signature via- # ``Signature.clone().set(headers=...)`` rather than- # in-place ``.set(...)`` on the caller's list. Cloning- # avoids cross-tenant header leakage if a future retry- # path or signature cache ever re-uses the original- # ``header_tasks`` list with a different ``FairnessKey``.- if fairness_headers:- signed_header_tasks = [- task.clone().set(headers=fairness_headers) for task in header_tasks- ]- else:- signed_header_tasks = header_tasks-- result = chord(signed_header_tasks)(callback_signature)-- logger.info(- f"Barrier enqueued via CeleryChordBarrier — "- f"header_tasks={len(signed_header_tasks)}, "- f"callback={callback_task_name}, "- f"queue={callback_queue}"- )-- return result-- except Exception:- # ``logger.exception`` auto-attaches the traceback — needed- # for debugging broker outages / serialisation failures- # at the chord entry point. Add the readily-available- # context so a Sentry/log triage gets execution/pipeline/- # callback/queue identifiers alongside the stack.- execution_id = callback_kwargs.get("execution_id")- pipeline_id = callback_kwargs.get("pipeline_id")- logger.exception(- f"[exec:{execution_id}] [pipeline:{pipeline_id}] "- f"Failed to enqueue barrier "- f"(callback={callback_task_name}, queue={callback_queue}, "- f"header_tasks={len(header_tasks)})"- )- raisediff --git a/workers/queue_backend/pg_barrier.py b/workers/queue_backend/pg_barrier.pyindex d841ae0bc..03c822a8b 100644--- a/workers/queue_backend/pg_barrier.py+++ b/workers/queue_backend/pg_barrier.py@@ -1,44 +1,41 @@-"""PG fan-in barrier — Postgres substrate for the ``Barrier`` Protocol.--Third ``WORKER_BARRIER_BACKEND`` option (``pg``) alongside ``chord`` and-``redis``. Mirrors :class:`~queue_backend.redis_barrier.RedisDecrBarrier`-exactly — same ``enqueue`` signature, same ``BarrierHandle | None`` contract,-same fairness plumbing, same Celery-dispatched header tasks with-``.link``/``.link_error`` — but moves the **aggregation** ("wait for N tasks,-then fire the callback with their results") from a Redis ``DECR`` counter to a-Postgres row. Selected at runtime by ``queue_backend.get_barrier``; default-stays ``chord``.--**Why a Postgres substrate.** It lets an execution coordinate in the *same*-Postgres that holds the PG queue — no Redis (or RabbitMQ chord backend) needed-for the fan-in. The transport for the header tasks themselves is unchanged-(still Celery); only the coordination moves.+"""PG fan-in barrier — the ``Barrier`` implementation.++Coordinates "wait for N header tasks, then fire the callback with their+results" on a Postgres row (``pg_barrier_state``) rather than a Celery chord or+a Redis ``DECR`` counter. Both of those substrates were deleted with the Celery+transport (UN-4078); this is the only barrier.++**Why a Postgres substrate.** An execution coordinates in the *same* Postgres+that holds the PG queue, so the fan-in needs no second datastore — and, unlike a+broker-side or TTL'd-counter barrier, its state is a directly queryable row the+reaper can recover.**Wire model.**1. ``enqueue``: UPSERT one ``pg_barrier_state`` row (``remaining = N``,``results = []``, ``expires_at = now() + ttl``, ``last_progress_at = now()``)- — the UPSERT clears any stale- state from a prior run reusing the same ``execution_id``. Each header task is- dispatched with- ``.link(barrier_pg_decr_and_check)`` (success) and- ``.link_error(barrier_pg_abort)`` (failure).-2. Per-task success: ``barrier_pg_decr_and_check`` runs ONE atomic statement —+ — the UPSERT clears any stale state from a prior run reusing the same+ ``execution_id``. Each header task is then dispatched onto the PG queue with a+ ``_barrier_context`` kwarg carrying ``execution_id`` / ``batch_index`` /+ ``callback_descriptor``.+2. Per-task success: a PG-consumed task fires no Celery ``.link``, so it claims+ its batch and runs the decrement **in-body** (:func:`run_batch_with_barrier` →+ :func:`_barrier_pg_decrement`) — ONE atomic statement,``UPDATE … SET remaining = remaining - 1, results = results ||jsonb_build_array(result) … RETURNING remaining, results``. The row lockserialises concurrent decrements, so exactly one task observes ``remaining =- 0``; that task dispatches the callback with the aggregated results, then- deletes the row. (No Lua — a single ``UPDATE … RETURNING`` is atomic in+ 0``; that task self-chains the callback onto PG with the aggregated results,+ then deletes the row. (No Lua — a single ``UPDATE … RETURNING`` is atomic inPostgres. The guarantee relies on each decrement committing in its owntransaction — do NOT batch decrements into a shared transaction, or the rowlock would hold and the serialisation that makes exactly one see 0 breaks.)-3. Per-task failure: ``barrier_pg_abort`` runs as a ``link_error``. It tears the- barrier down with ONE atomic statement (``DELETE … RETURNING`` in a single+3. Per-task failure: :func:`barrier_pg_abort`, called in-body, tears the barrier+ down with ONE atomic statement (``DELETE … RETURNING`` in a singletransaction): the row's existence is the dedup token, so N concurrentfailures collapse to a single cleanup, and a crash mid-abort rolls back(leaving the row for a sibling to retry) — there is no claimed-but-not-deleted- window. Mirrors chord's default error semantic (callback not invoked on- header failure).+ window. Preserves chord's old error semantic (callback not invoked on header+ failure).4. Stuck bound: ``last_progress_at`` is re-stamped to ``now()`` on enqueue AND onevery decrement, tracking when a batch last completed. The:mod:`~queue_backend.pg_queue.reaper` marks the stranded execution ERROR once@@ -57,9 +54,9 @@ succeeded. A failed task runs the abort (which deletes the row) instead of adecrement, so the count never reaches 0 and a late in-flight decrement finds norow → abandons. A decrement that drives ``remaining`` negative (expiry/replay)also cleans up without firing. The callback is dispatched BEFORE the row is-deleted, so a callback ``apply_async`` failure leaves the row (and its expiry)-in place rather than stranding the execution; the post-dispatch delete is-best-effort (logged, not raised) since the callback has already fired.+deleted, so a callback dispatch failure leaves the row (and its expiry) in place+rather than stranding the execution; the post-dispatch delete is best-effort+(logged, not raised) since the callback has already fired."""from __future__ import annotations@@ -77,11 +74,7 @@ import psycopg2import psycopg2.errorsimport psycopg2.extensions-from unstract.core.data_models import (- DEFAULT_WORKFLOW_TRANSPORT,- WorkflowTransport,- is_pg_transport,-)+from unstract.core.data_models import LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUEfrom .barrier import (BarrierContext,@@ -550,11 +543,35 @@ class _PgBarrierHandle:id: str+def build_callback_descriptor(+ *,+ task_name: str,+ kwargs: dict[str, Any],+ queue: str,+ fairness_headers: dict[str, Any] | None,+) -> CallbackDescriptor:+ """Build the descriptor carried to the consumer that fires the callback.++ A function rather than an inline literal so the rolling-deploy shim it stamps+ (:data:`~unstract.core.data_models.LEGACY_TRANSPORT_KEY`) is pinned by a test+ that needs no Postgres — the enqueue path's own tests are skipped wherever a+ database is unavailable, which would let the shim be dropped against a green+ required build. See ``tests/test_legacy_transport_shim.py``.+ """+ return {+ "task_name": task_name,+ "kwargs": kwargs,+ "queue": queue,+ "fairness_headers": fairness_headers,+ # Rolling-deploy shim: see LEGACY_TRANSPORT_KEY.+ LEGACY_TRANSPORT_KEY: LEGACY_TRANSPORT_VALUE,+ }++class PgBarrier:"""``Barrier`` implementation via a Postgres ``pg_barrier_state`` row.- Drop-in for ``CeleryChordBarrier`` / ``RedisDecrBarrier`` from the call- sites' perspective. See the module docstring for the wire model and the+ The only implementation. See the module docstring for the wire model and thefailure-masking guards."""@@ -567,7 +584,6 @@ class PgBarrier:callback_queue: str,app_instance: Any,fairness: FairnessKey | None = None,- transport: str = DEFAULT_WORKFLOW_TRANSPORT,) -> BarrierHandle | None:"""See :class:`queue_backend.barrier.Barrier.enqueue`.@@ -575,16 +591,11 @@ class PgBarrier:any substrate failure raises. ``app_instance`` is accepted for Protocolparity but unused.- ``transport`` selects the fan-out mode:-- - ``celery`` (default, the ``WORKER_BARRIER_BACKEND=pg`` legacy path):- header tasks are Celery-dispatched with ``.link(barrier_pg_decr_and_check)``- / ``.link_error(barrier_pg_abort)``; the link drives the decrement.- - ``pg_queue`` (the fire-and-forget path): header tasks are dispatched onto- the PG queue (``dispatch(backend=PG)``) with a ``_barrier_context`` kwarg- carrying ``execution_id`` / ``batch_index`` / ``callback_descriptor`` —- a PG-consumed task fires no ``.link``, so it claims its batch and runs- the decrement in-body, self-chaining the callback at ``remaining`` → 0.+ Header tasks are dispatched onto the PG queue (``dispatch(backend=PG)``)+ with a ``_barrier_context`` kwarg carrying ``execution_id`` /+ ``batch_index`` / ``callback_descriptor``. A PG-consumed task fires no+ Celery ``.link``, so it claims its batch and runs the decrement in-body,+ self-chaining the callback at ``remaining`` → 0."""del app_instance # Protocol parity; callback dispatched by the decrement.if not header_tasks:@@ -604,19 +615,14 @@ class PgBarrier:)execution_id = str(execution_id)- is_pg = is_pg_transport(transport)try:fairness_headers = fairness.as_header() if fairness else None- callback_descriptor: CallbackDescriptor = {- "task_name": callback_task_name,- "kwargs": callback_kwargs,- "queue": callback_queue,- "fairness_headers": fairness_headers,- }- if is_pg:- # The decrement (run in-body on the PG path) reads this to- # self-chain the callback onto PG rather than Celery.- callback_descriptor["transport"] = WorkflowTransport.PG_QUEUE.value+ callback_descriptor = build_callback_descriptor(+ task_name=callback_task_name,+ kwargs=callback_kwargs,+ queue=callback_queue,+ fairness_headers=fairness_headers,+ )# expires_at = absolute orphan cap (6h). last_progress_at = now() (the# reaper's fast stuck signal, re-stamped on every decrement).ttl_seconds = barrier_ttl_seconds()@@ -676,15 +682,13 @@ class PgBarrier:self._dispatch_headers(header_tasks,- is_pg=is_pg,execution_id=execution_id,callback_descriptor=callback_descriptor,fairness=fairness,- fairness_headers=fairness_headers,)logger.info(- f"Barrier enqueued via PgBarrier ({transport}) — "+ f"Barrier enqueued via PgBarrier — "f"exec_id={execution_id}, header_tasks={len(header_tasks)}, "f"callback={callback_task_name}, queue={callback_queue}")@@ -704,45 +708,28 @@ class PgBarrier:self,header_tasks: list[Signature],*,- is_pg: bool,execution_id: str,callback_descriptor: CallbackDescriptor,fairness: FairnessKey | None,- fairness_headers: dict[str, Any] | None,) -> None:- """Dispatch the N header tasks, one transport or the other.-- PG path (``is_pg``) → fire-and-forget onto the PG queue via- :meth:`_dispatch_header_pg` (no ``.link``). Celery path → ``.link`` /- ``.link_error`` chord-style. On any mid-loop dispatch failure, ``i`` of N- never reached the queue so the counter can't reach 0 — delete the barrier- row (and, on the PG path, reclaim dedup markers an earlier header may have- committed, since the in-flight ``barrier_pg_abort`` is a no-op once the row- is gone) so an in-flight decrement finds nothing, then re-raise.+ """Dispatch the N header tasks fire-and-forget onto the PG queue.++ On any mid-loop dispatch failure, ``i`` of N never reached the queue so+ the counter can't reach 0 — delete the barrier row and reclaim the dedup+ markers an earlier header may have committed (the in-flight+ ``barrier_pg_abort`` is a no-op once the row is gone) so an in-flight+ decrement finds nothing, then re-raise."""- link_signature = barrier_pg_decr_and_check.s(- execution_id=execution_id, callback_descriptor=callback_descriptor- )- link_error_signature = barrier_pg_abort.s(execution_id=execution_id)for i, task in enumerate(header_tasks):try:- if is_pg:- self._dispatch_header_pg(- task, i, execution_id, callback_descriptor, fairness- )- else:- cloned = task.clone()- if fairness_headers:- cloned.set(headers=fairness_headers)- cloned.link(link_signature)- cloned.link_error(link_error_signature)- cloned.apply_async()+ self._dispatch_header_pg(+ task, i, execution_id, callback_descriptor, fairness+ )except Exception:with contextlib.suppress(Exception):_delete_barrier(execution_id)- if is_pg:- with contextlib.suppress(Exception):- clear_execution_batches(execution_id)+ with contextlib.suppress(Exception):+ clear_execution_batches(execution_id)logger.exception(f"[exec:{execution_id}] header dispatch failed at task "f"{i}/{len(header_tasks)}; barrier row deleted to prevent "@@ -820,53 +807,28 @@ def _fire_barrier_callback() -> str:"""Dispatch the aggregating callback when the barrier completes; return its id.- Two transports, selected by the descriptor's ``transport`` marker:-- - ``"pg_queue"`` (fire-and-forget PG path): self-chain the callback onto- the PG queue via :func:`_dispatch_pg`, carrying the producer's fairness- (reconstructed from the stored headers) so the callback rides the same- org/priority as the Celery path — no Celery, so the whole execution stays- off the broker.- - absent / anything else (legacy ``.link`` path): dispatch via- ``current_app.signature(...).apply_async()`` — byte-identical to the legacy path,- preserving the ``fairness_headers`` the producer attached.-- On the PG path the callback kwargs carry :data:`PG_TRANSPORT_CALLBACK_KWARG`- so the aggregating callback can PG-gate its at-least-once duplicate guard (the- callback is unguarded against the redelivery its own dispatch admits; the- Celery ``.link`` path never injects it, so the guard is a no-op there).- """- if is_pg_transport(callback_descriptor.get("transport")):- # Copy (don't mutate the shared descriptor) + tag the PG transport so the- # callback can gate its duplicate guard on it.- pg_kwargs = {- **callback_descriptor["kwargs"],- PG_TRANSPORT_CALLBACK_KWARG: True,- }- handle = _dispatch_pg(- callback_descriptor["task_name"],- args=[all_results],- kwargs=pg_kwargs,- queue=callback_descriptor["queue"],- fairness=_fairness_from_headers(callback_descriptor.get("fairness_headers")),- )- return str(handle.id)-- from celery import current_app+ Self-chains the callback onto the PG queue via :func:`_dispatch_pg`, carrying+ the producer's fairness (reconstructed from the stored headers) so the+ callback rides the same org/priority as its header tasks did.- # Build the callback-signature kwargs explicitly (headers only when truthy) —- # matching CeleryChordBarrier's idiom, no spurious headers=None.- signature_kwargs: dict[str, Any] = {- "args": [all_results],- "kwargs": callback_descriptor["kwargs"],- "queue": callback_descriptor["queue"],+ The callback kwargs carry :data:`PG_TRANSPORT_CALLBACK_KWARG` so the+ aggregating callback can gate its at-least-once duplicate guard — it is+ otherwise unguarded against the redelivery its own dispatch admits.+ """+ # Copy (don't mutate the shared descriptor) + tag the transport so the+ # callback can gate its duplicate guard on it.+ pg_kwargs = {+ **callback_descriptor["kwargs"],+ PG_TRANSPORT_CALLBACK_KWARG: True,}- if callback_descriptor.get("fairness_headers"):- signature_kwargs["headers"] = callback_descriptor["fairness_headers"]- callback_signature = current_app.signature(- callback_descriptor["task_name"], **signature_kwargs+ handle = _dispatch_pg(+ callback_descriptor["task_name"],+ args=[all_results],+ kwargs=pg_kwargs,+ queue=callback_descriptor["queue"],+ fairness=_fairness_from_headers(callback_descriptor.get("fairness_headers")),)- return str(callback_signature.apply_async().id)+ return str(handle.id)class _DecrementRow(NamedTuple):@@ -907,12 +869,11 @@ def _apply_decrement(that default at its source.)- **COMMIT phase** — ``conn.commit()``. A failure here is AMBIGUOUS: theserver may have applied the commit before the socket dropped. Re-applying- could double-count, so it is NEVER retried — it propagates. On the in-body- PG path (:func:`run_batch_with_barrier`) the caller then tears the barrier- down so the execution fails fast; on the Celery ``.link`` path- (:func:`barrier_pg_decr_and_check`, ``max_retries=0``) it is not retried and- the barrier is reclaimed at ``expires_at`` by the reaper. Either way the- counter is never corrupted.+ could double-count, so it is NEVER retried — it propagates. The in-body+ caller (:func:`run_batch_with_barrier`) then tears the barrier down so the+ execution fails fast, and a barrier left behind by a hard crash is+ reclaimed at ``expires_at`` by the reaper. Either way the counter is never+ corrupted.Any non-connection error (e.g. the NUL-byte ``psycopg2.DataError`` from thejsonb cast) also propagates unchanged — only the idle-reap self-heals. This is@@ -1013,22 +974,22 @@ def _barrier_pg_decrement(Appends this task's result and decrements ``remaining`` in one atomicstatement; the single caller that drives ``remaining`` to 0 dispatches the- aggregating callback (then deletes the row). This is the plain, in-body-- callable core shared by two entry points:-- - the Celery ``link`` callback :func:`barrier_pg_decr_and_check` (today's- only caller, behaviour unchanged); and- - the PG-consumed pipeline path, which will call this directly in- the header task's body — a PG-consumed task fires no ``.link``, so the- decrement must run in-body.+ aggregating callback (then deletes the row). Called directly from the header+ task's body by :func:`run_batch_with_barrier` — a PG-consumed task fires no+ Celery ``.link``, so the decrement must run in-body. (It used to have a+ second entry point, a ``@worker_task`` link wrapper, deleted with the Celery+ transport in UN-4078.)Callers MUST run each decrement in its own committed transaction and MUST NOTreplay a *committed* decrement: re-applying one that already landed corruptsthe count (fires the callback early with incomplete results, or skips past 0and strands the barrier). This is enforced loudly, not just in prose — entryraises if the shared connection is already mid-transaction (see the guard- below), and the Celery wrapper pins ``max_retries=0`` so a task-level replay- can't re-drive it; an orphaned barrier is bounded by ``expires_at`` instead.+ below). The task-level replay guard that used to back this up — the Celery+ wrapper's ``max_retries=0`` — went with ``barrier_pg_decr_and_check`` in+ UN-4078; the decrement now runs in-body, so a replay would be a redelivery of+ the batch itself, which ``claim_batch`` already makes idempotent. An orphaned+ barrier is bounded by ``expires_at``.The decrement DOES self-heal one narrow, provably-safe case via:func:`_apply_decrement`: an execute-phase failure on a cached connection that@@ -1163,25 +1124,6 @@ def _barrier_pg_decrement(}-@worker_task(name="barrier_pg_decr_and_check", max_retries=0)-def barrier_pg_decr_and_check(- result: Any,- *,- execution_id: str,- callback_descriptor: CallbackDescriptor,-) -> dict[str, Any]:- """Per-task ``link`` callback for :class:`PgBarrier` (Celery entry point).-- Thin ``@worker_task`` wrapper around :func:`_barrier_pg_decrement` so the- decrement logic stays callable in-body (no ``.link``) on the PG-consumed- pipeline path. ``max_retries=0`` — a Celery retry would replay- the decrement and corrupt the count (see the core's contract).- """- return _barrier_pg_decrement(- result, execution_id=execution_id, callback_descriptor=callback_descriptor- )--@worker_task(name="barrier_pg_abort", max_retries=0)def barrier_pg_abort(request: Any = None,diff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.pyindex b43438c44..9d9c41cd8 100644--- a/workers/queue_backend/pg_queue/reaper.py+++ b/workers/queue_backend/pg_queue/reaper.py@@ -46,10 +46,14 @@ standby. A standby tries to acquire each cycle. The cycle interval MUST beshorter than the lease window, or the leader would lose the lease betweenrenews — enforced in :meth:`PgReaper.__init__`.-**Ships dark.** Launched explicitly (``python -m queue_backend.pg_queue.reaper``-or, later, ``run-worker.sh``); never part of the default worker set. With-``WORKER_BARRIER_BACKEND`` left at ``chord`` (default) there are no-``pg_barrier_state`` rows, so the sweep is a no-op until the PG barrier is used.+**A hard dependency, not an optional sweep.** This used to ship dark: the PG+barrier was one of three selectable substrates, so with ``WORKER_BARRIER_BACKEND``+at its ``chord`` default there were no ``pg_barrier_state`` rows and nothing to+reap. That is no longer true. PG is the only transport and the only barrier, and+this process additionally owns queue re-arm (crash redelivery) and scheduled+promotion (delayed delivery) — if it is not running, crashed work is never+redelivered and delayed messages never fire. Deployed as ``worker-pg-reaper``;+alert on its liveness."""from __future__ import annotationsdiff --git a/workers/queue_backend/redis_barrier.py b/workers/queue_backend/redis_barrier.pydeleted file mode 100644index 2c1e2fb3e..000000000--- a/workers/queue_backend/redis_barrier.py+++ /dev/null@@ -1,761 +0,0 @@-"""Redis-DECR Barrier — alternative substrate to ``CeleryChordBarrier``.--Implements the ``Barrier`` Protocol via the labs-design distributed-counter-pattern (``DECR remaining:{exec_id}``) with per-task result aggregation-in a Redis list. Selected at runtime by the ``WORKER_BARRIER_BACKEND``-env flag (default ``chord`` — see ``queue_backend.get_barrier``).--**Why this substrate exists.** Celery's ``chord(header)(body)`` aggregator-is the highest-risk Celery construct at our scale — documented silent-task drops at ~130K-task scale (see PG Queue decision journey). This-substrate replaces *only* the aggregation primitive; everything else-(dispatch, retries, result backend, monitoring) still rides on Celery.-The transport is unchanged — only the "wait for N tasks to complete,-then fire the callback with their results" coordination is moved from-Celery's chord backend to Redis.--**Wire model.**--1. ``enqueue``: ``SET remaining:{exec_id} N`` with TTL (default 6h,- tunable via ``WORKER_BARRIER_KEY_TTL_SECONDS``). The results list- is created lazily by the first link task's ``RPUSH`` — its TTL is- set inside the Lua script after the push, so both keys share the- same expiry window. Each header task is dispatched with- ``.link(barrier_decr_and_check.s(...))`` (success) and- ``.link_error(barrier_abort.s(...))`` (failure).-2. Per-task success: the link task runs ``RPUSH results + EXPIRE +- DECR remaining`` atomically via a Lua script. If the post-decrement- counter reads exactly 0, the link reads the full results list,- dispatches the callback with it as the first arg, then deletes- the Redis keys.-3. Per-task failure: ``barrier_abort`` runs as a link_error, DELing- both barrier keys + logging the failure. Mirrors Celery chord's- default error-propagation semantic (callback isn't invoked when- any header fails). The outer task's error handler marks the- workflow FAILED; this task only cleans barrier state.-4. TTL safety net: if the link/link_error never fire for some task- (worker crash mid-execution, broker outage), TTL expires the keys- after the configured window. Subsequent successful tasks' DECR on- the missing counter returns negative; the Lua script's ``< 0``- branch DELs cleanly without firing a spurious callback.--**Result-aggregation parity.** The callback receives-``list[BatchExecutionResult]`` exactly as it does today under-``CeleryChordBarrier`` — zero callback-side changes. Serialisation-rides through ``BatchExecutionResult.to_dict()`` / ``from_dict()``-(the typed callback boundary) ↔ JSON in Redis.--**Atomicity.** The per-task ``RPUSH + DECR`` is atomic via Lua — a-network partition mid-step can't split-brain the counter and the-result list. ``DECR`` is also atomic on its own, so exactly one task-sees the post-decrement 0.--**Mixed-version rolling deploy.** The ``WORKER_BARRIER_BACKEND`` env-var is read at worker startup, so each pod independently picks its-substrate. A given execution's tasks all share the publishing pod's-substrate choice (the barrier is selected once per execution start);-no cross-substrate aggregation within a single execution.-"""--from __future__ import annotations--import contextlib-import json-import logging-import os-from dataclasses import dataclass-from typing import TYPE_CHECKING, Any--from celery.canvas import Signature--from unstract.core.cache.redis_client import create_redis_client-from unstract.core.data_models import DEFAULT_WORKFLOW_TRANSPORT--from .barrier import (- _DEFAULT_BARRIER_TTL_SECONDS,- CallbackDescriptor,- barrier_ttl_seconds,-)-from .decorator import worker_task-from .fairness import FairnessKey-from .handle import BarrierHandle--if TYPE_CHECKING:- import redis as redis_lib--logger = logging.getLogger(__name__)---# Env-var prefix for the barrier's Redis connection. Falls back to-# ``REDIS_`` (the canonical worker Redis instance) via ``create_redis_client``'s-# nested-getenv pattern. Operators can point the barrier at a separate-# Redis by setting ``WORKER_BARRIER_REDIS_HOST`` / ``_PORT`` etc.-_BARRIER_REDIS_ENV_PREFIX = "WORKER_BARRIER_REDIS_"--# Belt-and-suspenders cleanup for orphaned ``remaining`` / ``results``-# keys. On the happy path the Lua script ``DEL``s both keys when-# ``remaining`` reaches 0 — TTL only matters when the counter stalls-# (e.g. some header tasks fail and the link never fires for them).-#-# **Must be strictly > longest plausible execution wall-clock.** If-# TTL expires while tasks are still pending, the next successful-# task's ``DECR`` creates the counter at ``-1`` and the script fires-# the callback with just that task's result — followed by more-# spurious callbacks as remaining tasks complete.-#-# Reference budget: the operator-tuned ceiling is whatever-# ``FILE_PROCESSING_TASK_TIME_LIMIT`` is set to in the deployment env-# (workers/sample.env ships 7200/2h; docker/sample.env up to 10800/3h).-# ``process_file_batch`` has ``max_retries=0`` so no retry amplification —-# worst case per batch is the time-limit ceiling. Multi-batch-# executions ≈ max(per-batch time) since batches run in parallel-# (worker concurrency is the limiter, not serial wait).-#-# 6h default gives 2-3x margin over the documented per-batch ceiling-# while staying in the same order of magnitude as-# ``FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000`` (5h). Operators with-# longer workflows (e.g. multi-step pipelines with chained barriers)-# or shorter known max-execution-time should tune via-# ``WORKER_BARRIER_KEY_TTL_SECONDS``.-# TTL lives in barrier.py (shared with PgBarrier — both bound an orphaned-# barrier by the same env var). Aliased so this module's internal call sites and-# tests keep their names.-_key_ttl_seconds = barrier_ttl_seconds-_KEY_TTL_DEFAULT_SECONDS = _DEFAULT_BARRIER_TTL_SECONDS---# Atomic ``RPUSH + EXPIRE + DECR``: returns ``(remaining, results)``.-#-# Three branches the Python side reads off the returned counter:-#-# remaining > 0 — pending, no results returned (still aggregating)-# remaining == 0 — complete, all results returned (we are the last).-# Keys are NOT DEL'd here — Python defers the DEL-# until after ``apply_async()`` succeeds, so that-# a callback-dispatch failure leaves the keys (and-# their TTL) in place rather than stranding the-# execution with no state + no recovery path.-# remaining < 0 — TTL-expired counter (or replay after cleanup);-# keys DEL'd inside the script to prevent further-# spurious fires, no callback dispatched.-#-# Setting the ``results`` key TTL inside the script — ``EXPIRE`` on-# the call AFTER ``RPUSH`` ensures the key exists when EXPIRE runs-# (issuing EXPIRE before any push is a no-op on a non-existent key,-# which is the bug the previous version had). Repeated EXPIRE per-# RPUSH is cheap and keeps both keys' TTLs aligned.-_RPUSH_DECR_LUA = """-local remaining_key = KEYS[1]-local results_key = KEYS[2]-local result_json = ARGV[1]-local ttl_seconds = tonumber(ARGV[2])-redis.call("RPUSH", results_key, result_json)-redis.call("EXPIRE", results_key, ttl_seconds)-local remaining = redis.call("DECR", remaining_key)-if remaining == 0 then- -- We're the last task. LRANGE the results for the Python side to- -- dispatch the callback. DO NOT DEL the keys here — Python defers- -- the DEL until after apply_async() succeeds so a dispatch failure- -- (broker outage, serialisation error) leaves the keys + TTL in- -- place. Without this deferral, an apply_async failure would- -- strand the execution with both keys gone, no TTL, no Celery- -- retry (max_retries=0), and no link_error fallback — strictly- -- worse than the chord baseline (where Celery's chord backend- -- owns + retries body invocation).- local all_results = redis.call("LRANGE", results_key, 0, -1)- return {remaining, all_results}-end-if remaining < 0 then- -- Counter was TTL-expired (or already cleaned up) when this task- -- ran. ``DECR`` on a missing key created it at ``-1``; subsequent- -- tasks would land here too. DEL both keys so neither this task- -- nor any subsequent one fires a spurious callback; the abandoned- -- branch on the Python side surfaces an error-severity log.- redis.call("DEL", remaining_key, results_key)- return {remaining, {}}-end-return {remaining, {}}-"""---# Process-local cached Redis client. Lazy-initialised on first call-# within a worker process; reused across every ``barrier_decr_and_check``-# / ``barrier_abort`` / ``RedisDecrBarrier`` invocation in that process.-#-# Why a singleton: ``create_redis_client`` builds a fresh-# ``ConnectionPool`` each call. Without caching, a 1000-task fan-out-# would spin up 1000 separate pools (1000 TCP handshakes torn down in-# sequence) — exactly the scale this substrate exists to handle.-#-# Fork safety: Celery's prefork model re-imports this module in each-# child process, so each worker process gets its own fresh-# ``_redis_client_singleton`` — no socket sharing across fork-# boundaries (which would corrupt state).-#-# Thread safety: redis-py's ``ConnectionPool`` is internally thread-safe,-# so concurrent users of the cached client are fine. The lazy-init-# itself is not locked — two threads racing on the first call could-# briefly create two pools, with one being orphaned. The cost is one-# short-lived pool (immediately GC'd), not a correctness issue.-_redis_client_singleton: redis_lib.Redis | None = None---def _get_redis_client() -> redis_lib.Redis:- """Return a process-cached Redis client for the barrier.-- ``create_redis_client``'s nested-getenv fallback only covers- HOST/PORT/PASSWORD/USER/DB — it does NOT cross-fall-back- SENTINEL_MODE or SSL. In a deployment where the canonical Redis- is Sentinel-backed or TLS-secured, leaving ``WORKER_BARRIER_REDIS_*``- unset and relying on the documented HOST fallback would result in- the barrier inheriting the right host/port but connecting- standalone/plaintext — which fails (or worse, connects without- TLS) every execution the moment the flag flips.-- To avoid that footgun: when no barrier-specific HOST is set, use- the canonical ``REDIS_`` prefix *directly* so we inherit the FULL- canonical config (including Sentinel + SSL). When an operator- has set ``WORKER_BARRIER_REDIS_HOST`` they've opted into a- dedicated barrier Redis and are responsible for the full config- (HOST + PORT + ... + SENTINEL_MODE + SSL).-- ``decode_responses=True`` so ``LRANGE`` returns ``list[str]``- (we JSON-decode each entry).- """- global _redis_client_singleton- if _redis_client_singleton is None:- # If no dedicated barrier-Redis host is configured, use the- # canonical REDIS_ prefix so Sentinel/SSL config inherits too.- if os.getenv(f"{_BARRIER_REDIS_ENV_PREFIX}HOST"):- env_prefix = _BARRIER_REDIS_ENV_PREFIX- else:- env_prefix = "REDIS_"- _redis_client_singleton = create_redis_client(- env_prefix=env_prefix,- decode_responses=True,- socket_timeout=5,- socket_connect_timeout=5,- )- return _redis_client_singleton---def _remaining_key(execution_id: str) -> str:- return f"barrier:remaining:{execution_id}"---def _results_key(execution_id: str) -> str:- return f"barrier:results:{execution_id}"---def _abort_lock_key(execution_id: str) -> str:- """Per-execution dedup lock for ``barrier_abort``.-- Set via ``SET NX EX`` in ``barrier_abort`` to collapse N concurrent- header-task failures into a single Sentry/error event. MUST be- cleared at the start of a new barrier (in ``enqueue``) — otherwise- a retry that reuses an ``execution_id`` would hit a stale lock,- early-exit the abort path, and silently mask the failed retry as- successful.- """- return f"barrier:abort_lock:{execution_id}"---class RedisDecrBarrier:- """``Barrier`` implementation via Redis ``DECR remaining`` pattern.-- Drop-in replacement for ``CeleryChordBarrier`` from the call- sites' perspective — same ``enqueue(header_tasks, ...)`` signature,- same ``BarrierHandle | None`` return contract, same fairness- plumbing. The substrate difference is invisible above the- ``Barrier`` Protocol.-- See module docstring for the wire model, atomicity, and failure- semantics.- """-- def __init__(self, redis_client: redis_lib.Redis | None = None) -> None:- """Args:- redis_client: Optional pre-built client. Tests pass a fake- (``fakeredis``) here. Production leaves it ``None`` so- the barrier builds its own from env on first use.- """- self._redis_client = redis_client-- @property- def redis(self) -> redis_lib.Redis:- """Lazy-init the Redis client.-- Built on first use rather than at construction time so the- module can be imported (and the worker task registered) in- contexts where Redis isn't yet reachable (e.g. test collection,- worker boot before networking up).-- Production path: ``self._redis_client is None`` → falls back to- (and caches) the module-level ``_redis_client_singleton``, so- the instance and the standalone ``barrier_decr_and_check`` /- ``barrier_abort`` worker tasks all share the same- ``ConnectionPool``.-- Test path: ``__init__(redis_client=...)`` bypasses this branch- entirely — fixtures inject a ``MagicMock`` / ``fakeredis``- without touching the module-level singleton (which would leak- across tests).- """- if self._redis_client is None:- self._redis_client = _get_redis_client()- return self._redis_client-- def enqueue(- self,- header_tasks: list[Signature],- *,- callback_task_name: str,- callback_kwargs: dict[str, Any],- callback_queue: str,- app_instance: Any,- fairness: FairnessKey | None = None,- transport: str = DEFAULT_WORKFLOW_TRANSPORT,- ) -> BarrierHandle | None:- """See :class:`queue_backend.barrier.Barrier.enqueue`.-- ``transport`` is accepted for Protocol parity and ignored — the Redis- barrier is the Celery-transport fan-in substrate (the PG transport uses- ``PgBarrier``'s fire-and-forget mode). Mirrors ``CeleryChordBarrier``:-- - Empty ``header_tasks`` → ``None`` (caller handles the- zero-files contract); no Redis keys touched.- - Any substrate-level failure (Redis down, dispatch error,- serialisation issue) → raises.- - Success → returns a ``BarrierHandle`` whose ``.id`` is the- execution id (no ``AsyncResult`` since there's no Celery- chord wrapping the group; the id is what the call site- logs for chord-id traceability).-- Note: ``app_instance`` is accepted for ``Barrier`` Protocol- parity with ``CeleryChordBarrier`` but unused by this- substrate. The callback signature is built inside the link- task via ``celery.current_app.signature(...)`` (so the link- runs against the *worker's* app, not the producer's). Keeping- the parameter in the signature lets call sites swap between- substrates without per-substrate adapter code.- """- # Explicit ``del`` documents the Protocol-parity intent and- # satisfies static linters' unused-parameter check.- del app_instance, transport- if not header_tasks:- execution_id = callback_kwargs.get("execution_id")- pipeline_id = callback_kwargs.get("pipeline_id")- logger.info(- f"[exec:{execution_id}] [pipeline:{pipeline_id}] "- "Zero header tasks detected — skipping barrier enqueue "- "(parent should handle pipeline status updates directly)"- )- return None-- execution_id = callback_kwargs.get("execution_id")- if not execution_id:- raise ValueError(- "RedisDecrBarrier requires execution_id in callback_kwargs — "- "it's the key suffix for the per-execution remaining/results "- "Redis keys"- )- execution_id = str(execution_id)-- try:- fairness_headers = fairness.as_header() if fairness else None-- # The link task body needs to know:- # - which execution's counter to decrement- # - how to reconstruct + dispatch the callback when count hits 0- # All Celery-serialisable so the link can run on any worker.- callback_descriptor: CallbackDescriptor = {- "task_name": callback_task_name,- "kwargs": callback_kwargs,- "queue": callback_queue,- "fairness_headers": fairness_headers,- }-- # Initialise the counter before any header task can fire.- # ``SET remaining N EX <ttl>`` overwrites any stale counter- # left from a prior run with the same exec_id; the upfront- # ``DEL results, abort_lock`` (below) clears the rest of- # the prior-run state (the results list grows via RPUSH so- # we must clear it explicitly; the abort_lock would otherwise- # silently mask a retry's failure as success — see the- # comment block at the DELETE call). ``ttl`` is- # ``_key_ttl_seconds()`` (6h default).- ttl_seconds = _key_ttl_seconds()- # Clear any leftover state from a prior execution with this- # execution_id (including the dedup lock written by a- # previous run's ``barrier_abort``). Without DELing the- # abort_lock here, a retry that reuses execution_id would- # find the stale lock and route every ``barrier_abort``- # straight to the deduplicated branch — leaving B's- # ``remaining``/``results`` keys uncleaned. In-flight- # successful tasks from B would then hit ``remaining == 0``- # via DECR and fire the callback with partial results,- # silently masking the failed retry as successful.- self.redis.delete(- _results_key(execution_id),- _abort_lock_key(execution_id),- )- self.redis.set(- _remaining_key(execution_id),- len(header_tasks),- ex=ttl_seconds,- )- # ``results`` key TTL is set inside the Lua script on- # first RPUSH — ``EXPIRE`` on a non-existent key is a- # no-op, so we can't pre-set it here.-- # Stamp fairness on each header task and attach the link- # (success) + link_error (failure) callbacks.- # ``Signature.clone()`` avoids mutating the caller's list- # (same reasoning as ``CeleryChordBarrier``).- link_signature = barrier_decr_and_check.s(- execution_id=execution_id,- callback_descriptor=callback_descriptor,- )- # link_error explicitly propagates header-task failures —- # mirrors Celery chord's default error semantic (chord- # callback isn't invoked on header failure). Without this,- # a failed header task would leave the counter stuck and- # the execution would hang until TTL cleanup. The abort- # task DELs the barrier keys + logs the error; the outer- # task's error path is responsible for the workflow- # status update.- link_error_signature = barrier_abort.s(execution_id=execution_id)- for i, task in enumerate(header_tasks):- try:- cloned = task.clone()- if fairness_headers:- cloned.set(headers=fairness_headers)- cloned.link(link_signature)- cloned.link_error(link_error_signature)- cloned.apply_async()- except Exception:- # Mid-loop dispatch failure: tasks 0..i-1 are- # already in flight with the link attached, but- # ``i`` of N never made it to the broker. The- # counter would never reach 0 → execution hangs.- # DEL the barrier keys so the in-flight links'- # DECR-on-missing returns negative and the Lua- # script's ``< 0`` branch DELs cleanly. Re-raise- # so the caller's status-update path marks the- # workflow ERROR.- self.redis.delete(- _remaining_key(execution_id),- _results_key(execution_id),- )- # ``logger.exception`` captures the underlying- # apply_async failure cause (broker timeout vs- # serialisation error vs routing error) alongside- # the orphan-count context — without it, a reader- # of this log can't tell why dispatch failed and- # has to cross-reference the outer handler's log.- logger.exception(- f"[exec:{execution_id}] apply_async failed at "- f"task {i}/{len(header_tasks)}; {i} orphan tasks "- f"already dispatched. Barrier keys DEL'd to "- f"prevent spurious callback fires from the "- f"orphan tasks' link decrements."- )- raise-- logger.info(- f"Barrier enqueued via RedisDecrBarrier — "- f"exec_id={execution_id}, "- f"header_tasks={len(header_tasks)}, "- f"callback={callback_task_name}, "- f"queue={callback_queue}"- )-- # Handle's ``.id`` is the execution id — same call sites- # log ``chord_id`` from this; the semantic shifts from- # "Celery chord aggregator task id" to "execution id" but- # both are stable per-execution identifiers and the log- # consumers don't depend on the underlying task vs exec id- # distinction.- return _RedisBarrierHandle(id=execution_id)-- except Exception:- execution_id = callback_kwargs.get("execution_id")- pipeline_id = callback_kwargs.get("pipeline_id")- logger.exception(- f"[exec:{execution_id}] [pipeline:{pipeline_id}] "- f"Failed to enqueue barrier via Redis "- f"(callback={callback_task_name}, queue={callback_queue}, "- f"header_tasks={len(header_tasks)})"- )- raise---@dataclass(frozen=True, slots=True)-class _RedisBarrierHandle:- """Minimal ``BarrierHandle`` implementation.-- Just an ``id`` attribute (the execution id) so call sites that- log ``handle.id`` for chord-id tracing keep working. A class- (rather than a ``namedtuple``) gives a clear home for the- docstring + ``slots=True`` memory control; the ``BarrierHandle``- Protocol is structural so any object exposing ``id: str`` would- type-check.- """-- id: str---@worker_task(- name="barrier_decr_and_check",- # ``max_retries=0`` — the link task is intentionally non-idempotent- # at the Redis-script level (RPUSH + DECR are committed atomically,- # but a Celery retry would replay the script and double-count). Any- # Celery-level retry would cause a second RPUSH + DECR, corrupting- # the aggregation. If the link fails after the Lua commit (e.g. the- # subsequent callback ``apply_async`` raises), the counter has- # already advanced — TTL-driven cleanup is the safety net, and the- # outer execution's error path marks the workflow status.- #- # Trade-off: a transient Redis blip on the link's Lua call means- # this task's contribution is lost (counter doesn't advance) and- # the execution hangs until TTL. Acceptable because (a) the- # alternative — letting Celery retry — corrupts the aggregation,- # and (b) Redis blips are operator-visible via monitoring, unlike- # silent double-counts.- max_retries=0,-)-def barrier_decr_and_check(- result: Any,- *,- execution_id: str,- callback_descriptor: CallbackDescriptor,-) -> dict[str, Any]:- """Per-task link callback for ``RedisDecrBarrier``.-- Runs after each header task completes successfully. Pushes the- task's result into the per-execution Redis list, decrements the- counter, and (if the post-decrement counter reads 0) dispatches- the aggregating callback with the full result list.-- ``result`` is whatever the header task returned — for ``process_file_batch``- that's a ``BatchExecutionResult`` already serialised to a dict via- the typed callback boundary. We pass it through ``json.dumps`` to- survive the Redis round-trip.- """- from celery import current_app-- try:- # No ``default=str`` — that would silently stringify any non-- # JSON-safe leaf (datetime, UUID, Decimal, sets...) and mask- # the typed-boundary regression the surrounding ``try/except``- # is here to surface. The header task's return value is- # expected to be a fully JSON-safe dict per the- # ``BatchExecutionResult.to_dict()`` contract; anything else- # should fail loudly here so the regression surfaces.- result_json = json.dumps(result)- except (TypeError, ValueError):- logger.exception(- f"[exec:{execution_id}] Header task result is not "- f"JSON-serialisable — barrier aggregation cannot proceed. "- f"This indicates a typed-boundary regression; the "- f"BatchExecutionResult.to_dict() contract must produce a "- f"JSON-safe shape."- )- raise-- redis_client = _get_redis_client()- script = redis_client.register_script(_RPUSH_DECR_LUA)- remaining, raw_results = script(- keys=[- _remaining_key(execution_id),- _results_key(execution_id),- ],- args=[result_json, _key_ttl_seconds()],- )-- logger.info(f"[exec:{execution_id}] Barrier link DECR → remaining={remaining}")-- if remaining > 0:- # More tasks pending — nothing to do.- return {"status": "pending", "remaining": remaining}-- if remaining < 0:- # TTL-expired counter (or replay after cleanup). The Lua- # script already DEL'd both keys to prevent further spurious- # fires on subsequent task completions. No callback dispatch.- #- # Logged at ERROR (not WARNING) because reaching this branch- # means the barrier was torn down out from under in-flight- # tasks — by definition an abnormal terminal state where the- # execution almost certainly did not complete normally.- # Returning normally from a .link is indistinguishable from- # success to Celery; the ERROR log is the only execution-id-- # tagged signal an operator can correlate to the hung/- # incorrectly-statused execution.- logger.error(- f"[exec:{execution_id}] Barrier abandoned — counter went "- f"negative (remaining={remaining}); keys were TTL-expired "- f"or torn down. No callback dispatched; execution likely "- f"in an inconsistent terminal state and needs investigation."- )- return {"status": "abandoned", "remaining": remaining}-- # remaining == 0: we're the last task. Dispatch the callback.- all_results = [json.loads(r) for r in raw_results]- callback_task_name = callback_descriptor["task_name"]- callback_kwargs = callback_descriptor["kwargs"]- callback_queue = callback_descriptor["queue"]- fairness_headers = callback_descriptor.get("fairness_headers")-- callback_signature = current_app.signature(- callback_task_name,- args=[all_results],- kwargs=callback_kwargs,- queue=callback_queue,- **({"headers": fairness_headers} if fairness_headers else {}),- )- # Dispatch FIRST; DEL the barrier keys only after dispatch succeeds.- #- # If apply_async raises (broker outage, serialisation error,- # routing failure), the exception propagates. The barrier keys- # are NOT yet DEL'd, so:- # - Subsequent late-arriving link tasks (if any) hit the Lua- # ``< 0`` branch (counter is still 0 — DECR-on-zero would- # yield -1) and exit cleanly via the abandoned branch.- # - The keys TTL-expire eventually — Redis cleans them up.- # - The link task's failure surfaces via Celery's standard- # task-failure channels.- #- # If we DEL'd before apply_async (the previous design), an- # apply_async failure would strand the execution with no keys,- # no TTL, no Celery retry (max_retries=0), and no link_error- # fallback — silent infinite hang. The deferred DEL is what- # restores parity-or-better with the chord baseline.- callback_result = callback_signature.apply_async()- redis_client.delete(- _remaining_key(execution_id),- _results_key(execution_id),- )-- logger.info(- f"[exec:{execution_id}] Barrier complete — "- f"fired callback {callback_task_name} on {callback_queue} "- f"with {len(all_results)} aggregated results "- f"(callback_task_id={callback_result.id})"- )-- return {- "status": "complete",- "callback_task_id": callback_result.id,- "aggregated_count": len(all_results),- }---@worker_task(- name="barrier_abort",- # Same posture as ``barrier_decr_and_check`` — non-idempotent at- # the Redis level (DEL is fine to repeat, but we don't want- # double-logged abort events polluting Sentry).- max_retries=0,-)-def barrier_abort(- request: Any = None,- exc: Any = None,- traceback: Any = None,- *,- execution_id: str,-) -> dict[str, Any]:- """``link_error`` callback: header task failed → clean up barrier state.-- Celery 5.5 invokes new-style errbacks as- ``errback(request, exc, traceback)`` only for ``bind=False`` tasks- with arity > 1. If a worker hits the ``NotRegistered`` fallback- during a mixed-version rolling deploy (or if this task is ever- switched to ``bind=True``), Celery calls the errback old-style- with just ``(task_id,)``. The defaults on ``request`` / ``exc`` /- ``traceback`` mean the old-style path degrades to a clean- ``exc=None`` log line rather than a confusing- ``TypeError: missing required positional arguments``.-- Mirrors Celery chord's default error semantic: when any header- task fails, the chord callback isn't invoked. Without explicit- cleanup the ``remaining`` counter would stay above 0 until TTL- expiry, the ``results`` key would leak, and any subsequent- successful header tasks' link would still DECR — eventually- spurious-firing the callback with partial results (or via the- Lua ``< 0`` branch, abandoning the execution but only after the- extra task work).-- We DEL the barrier keys here so the in-flight successful tasks'- link DECR returns negative → Lua ``< 0`` branch fires →- aggregation cleanly abandoned. **Terminal status drivers**: this- task does NOT mark the workflow FAILED — the outer orchestrators- (``_run_workflow_api`` / ``_orchestrate_file_processing_general``)- wrap only the *synchronous* fan-out in try/except and have- already returned by the time ``barrier_abort`` runs. Workflow- terminal status on async header failure is driven by per-file- updates inside ``process_file_batch`` plus the eventual- aggregating callback (which won't fire because we DEL'd the- keys). A future Phase 6b' refinement may have ``barrier_abort``- itself drive the execution-level status update.-- **Concurrent-failure dedup.** Every header task attaches the same- ``link_error``, so N simultaneous failures would otherwise fire N- ``barrier_abort`` executions — each calling ``logger.error`` and- each producing its own Sentry event for what is a *single* logical- execution failure. A ``SET NX`` lock on- ``barrier:abort_lock:{execution_id}`` makes the first abort win;- subsequent aborts for the same execution see the lock present and- early-exit silently. The DELs are idempotent anyway (DEL on a- missing key is a no-op), but the dedup collapses the alert noise.-- **Lock release on DELETE failure.** If SET NX succeeds (this- abort wins) but the subsequent DELETE then raises (mid-abort- Redis blip), the lock stays held while the barrier keys are NOT- deleted. Every sibling ``barrier_abort`` would then hit the- dedup early-exit and do nothing, leaving the counter alive for- in-flight successful tasks to DECR to 0 → callback fires with- partial results → silent success masking. To close this hole,- we explicitly release the abort_lock if the DELETE raises, so- a sibling abort can re-acquire it and complete the cleanup.- """- redis_client = _get_redis_client()- # ``SET NX EX`` — first-write-wins with auto-expiry. TTL matches- # the barrier keys' TTL so the lock cleans up alongside them.- # The lock is also explicitly DEL'd by ``enqueue`` at the start- # of any new execution with the same execution_id, to prevent a- # stale lock from masking a retry's failure as success.- if not redis_client.set(- _abort_lock_key(execution_id), "1", ex=_key_ttl_seconds(), nx=True- ):- # Another barrier_abort for this execution already won the- # SET NX race. Silently exit — no log, no Sentry, no- # duplicate DEL.- return {- "status": "deduplicated",- "execution_id": execution_id,- }- try:- redis_client.delete(- _remaining_key(execution_id),- _results_key(execution_id),- )- except Exception:- # DELETE failed AFTER we won the SET NX. Release the lock so- # subsequent aborts aren't dedup-suppressed into no-ops while- # the barrier keys still exist. ``suppress`` because if the- # lock-release also fails, the original DELETE exception is- # the more useful one to propagate (Redis is clearly in- # trouble and the original error captures that).- with contextlib.suppress(Exception):- redis_client.delete(_abort_lock_key(execution_id))- raise- logger.error(- f"[exec:{execution_id}] Header task failed; barrier aborted. "- f"Cleaned up remaining/results keys. exc={exc!r}"- )- return {- "status": "aborted",- "execution_id": execution_id,- "reason": str(exc),- }diff --git a/workers/scheduler/tasks.py b/workers/scheduler/tasks.pyindex b30742c45..d79f7bf75 100644--- a/workers/scheduler/tasks.py+++ b/workers/scheduler/tasks.py@@ -39,11 +39,10 @@ from shared.patterns.notification.helper import trigger_notificationfrom shared.utils.api_client_singleton import get_singleton_api_clientfrom unstract.core.data_models import (- DEFAULT_WORKFLOW_TRANSPORT,+ LEGACY_TRANSPORT_KEY,+ LEGACY_TRANSPORT_VALUE,NotificationPayload,NotificationSource,- is_pg_transport,- normalize_transport,)# Import the exact backend logic to ensure consistency@@ -159,22 +158,11 @@ def _execute_scheduled_workflow(pipeline_id=context.pipeline_id,)- # Transport this execution rides (9e), decided by the backend at creation- # and carried in the dispatched task's payload. Absent key (older backend)- # → celery default; a present-but-unrecognized value (version skew) is- # coerced to celery with a loud warning rather than dispatched onto an- # unknown substrate (fail-closed).- transport = normalize_transport(- workflow_execution.get("transport", DEFAULT_WORKFLOW_TRANSPORT),- logger=logger,- context=f" [exec:{execution_id}]",- )-logger.info(f"[exec:{execution_id}] [pipeline:{context.pipeline_id}] Created workflow execution for scheduled pipeline {context.pipeline_name}")- # Step 2: Trigger async workflow execution via direct Celery dispatch+ # Step 2: Trigger async workflow executionlogger.info(f"[exec:{execution_id}] [pipeline:{context.pipeline_id}] Triggering async execution for workflow {context.workflow_id}")@@ -196,13 +184,13 @@ def _execute_scheduled_workflow(kwargs={"use_file_history": context.use_file_history,"pipeline_id": context.pipeline_id,- "transport": transport,+ # Rolling-deploy shim: see LEGACY_TRANSPORT_KEY.+ LEGACY_TRANSPORT_KEY: LEGACY_TRANSPORT_VALUE,},queue=QueueName.GENERAL,- # Orchestrator transport (9e PR A / 2d): route async_execute_bin- # onto PG for a pg_queue execution (carried-marker wins over the- # allow-list); None keeps the legacy Celery dispatch.- backend=QueueBackend.PG if is_pg_transport(transport) else None,+ # Stated explicitly rather than left to select_backend(): the whole+ # execution pipeline is pinned to one substrate at its entry point.+ backend=QueueBackend.PG,fairness=FairnessKey(org_id=context.organization_id,workload_type=WorkloadType.NON_API,diff --git a/workers/shared/models/execution_models.py b/workers/shared/models/execution_models.pyindex ca5c33278..c24f7a534 100644--- a/workers/shared/models/execution_models.py+++ b/workers/shared/models/execution_models.py@@ -10,10 +10,8 @@ from typing import TYPE_CHECKING, Any# Import shared domain models from corefrom unstract.core.data_models import (- DEFAULT_WORKFLOW_TRANSPORT,ExecutionStatus,PreCreatedFileData,- normalize_transport,serialize_dataclass_to_dict,)@@ -390,11 +388,6 @@ class WorkflowContextData:settings: dict[str, Any] | None = Nonemetadata: dict[str, Any] | None = Noneis_scheduled: bool = False- # Transport this execution rides end-to-end (9e). Populated from the inbound- # task payload; the fan-out read that keeps the pipeline on one transport- # lands in PR 2. Defaults to legacy Celery; coerced fail-closed in- # __post_init__ so a garbage payload value can't reach the PR 2 read site.- transport: str = DEFAULT_WORKFLOW_TRANSPORTpre_created_file_executions: dict[str, PreCreatedFileData] = field(default_factory=dict)@@ -424,16 +417,6 @@ class WorkflowContextData:f"Must be one of: {valid_types}")- # Coerce transport fail-closed: a garbage payload value (version skew,- # bad PG JSONB) degrades to Celery here rather than reaching the PR 2- # fan-out read. Unlike workflow_type (no safe default → raise), transport- # is reversible by design, so it falls back instead of crashing.- self.transport = normalize_transport(- self.transport,- logger=logger,- context=f" (execution {self.execution_id})",- )-@propertydef file_count(self) -> int:"""Get the number of files in this workflow context."""diff --git a/workers/shared/models/file_processing.py b/workers/shared/models/file_processing.pyindex b634d795e..af4062417 100644--- a/workers/shared/models/file_processing.py+++ b/workers/shared/models/file_processing.py@@ -32,7 +32,6 @@ class FileProcessingContext:workflow_logger: Any = None, # Type as Any to avoid import dependencycurrent_file_idx: int = 1,total_files: int = 1,- transport: str | None = None,):self.file_data = file_dataself.file_hash = file_hash@@ -43,10 +42,6 @@ class FileProcessingContext:self.workflow_logger = workflow_loggerself.current_file_idx = current_file_idxself.total_files = total_files- # Execution transport (celery | pg_queue). Authoritative here because- # WorkerFileData carries no transport; threaded from the batch context so- # the destination layer can apply PG-only guards.- self.transport = transport# Extract common identifiersself.execution_id = file_data.execution_iddiff --git a/workers/shared/processing/files/processor.py b/workers/shared/processing/files/processor.pyindex 52c4b9a11..90009d3a5 100644--- a/workers/shared/processing/files/processor.py+++ b/workers/shared/processing/files/processor.py@@ -477,7 +477,6 @@ class FileProcessor:workflow_file_execution_id: str = None,workflow_file_execution_object: Any = None,workflow_logger: WorkerWorkflowLogger = None,- transport: str | None = None,) -> FileProcessingResult:"""Main orchestrator method that replaces the complex _process_file method.@@ -513,7 +512,6 @@ class FileProcessor:workflow_logger=workflow_logger,current_file_idx=current_file_idx,total_files=total_files,- transport=transport,)logger.debug(diff --git a/workers/shared/workflow/execution/orchestration_utils.py b/workers/shared/workflow/execution/orchestration_utils.pyindex 80bfa0bc2..d40ac3663 100644--- a/workers/shared/workflow/execution/orchestration_utils.py+++ b/workers/shared/workflow/execution/orchestration_utils.py@@ -9,14 +9,10 @@ from __future__ import annotationsimport osfrom typing import TYPE_CHECKING, Any-from queue_backend import BarrierHandle, FairnessKey, get_barrier+from queue_backend import BarrierHandle, FairnessKeyfrom queue_backend.pg_barrier import PgBarrier-from unstract.core.data_models import (- DEFAULT_WORKFLOW_TRANSPORT,- ExecutionStatus,- is_pg_transport,-)+from unstract.core.data_models import ExecutionStatusfrom ...enums import FileDestinationType, PipelineTypefrom ...enums.worker_enums import QueueName@@ -24,30 +20,9 @@ from ...infrastructure.logging import WorkerLoggerif TYPE_CHECKING:from celery.canvas import Signature- from queue_backend import Barrierlogger = WorkerLogger.get_logger(__name__)-# Single ``Barrier`` instance reused across all-# ``WorkflowOrchestrationUtils.create_chord_execution`` calls on the **celery**-# transport. The substrate is selected at module-import (worker startup) time by-# the ``WORKER_BARRIER_BACKEND`` env var (default ``chord``). Flag flips require a-# pod restart — same posture as every other ``WORKER_*`` env in the codebase.-_BARRIER = get_barrier()---def _barrier_for_transport(transport: str) -> Barrier:- """Pick the fan-in substrate for a per-execution transport (9e).-- The ``pg_queue`` transport always coordinates on Postgres — it uses a fresh- :class:`PgBarrier` in its fire-and-forget mode regardless of- ``WORKER_BARRIER_BACKEND`` (the env-selected singleton is the *celery*-transport- substrate, which may be ``chord``). Every other transport uses that singleton.- """- if is_pg_transport(transport):- return PgBarrier()- return _BARRIER-class WorkflowOrchestrationUtils:"""Centralized workflow orchestration patterns and utilities."""@@ -62,13 +37,9 @@ class WorkflowOrchestrationUtils:logger: Any,workflow_logger: Any | None = None,) -> None:- """Record a **PG** orchestration failure: surface the error to the UI and+ """Record an orchestration failure: surface the error to the UI andreconcile the file counters, both best-effort.- Call this ONLY on the PG transport (the caller gates on- ``is_pg_transport``); the Celery failure path keeps its own original- status update untouched, so it stays byte-identical.-Two things happen, neither of which may disturb the caller's control flow(it still has the original orchestration exception to log / re-raise):@@ -116,15 +87,14 @@ class WorkflowOrchestrationUtils:app_instance: Any,*,fairness: FairnessKey | None = None,- transport: str = DEFAULT_WORKFLOW_TRANSPORT,) -> BarrierHandle | None:- """Standardized fan-out + callback pattern (Phase 6 ``Barrier``).+ """Standardized fan-out + callback pattern.- Routes through ``CeleryChordBarrier`` — a thin wrapper around- ``celery.chord(header)(body)`` that lets Phase 6b swap the- substrate (e.g. to ``RedisDecrBarrier``) without touching this- call site a second time. Behaviour is identical to the- previous direct ``chord(...)`` call.+ Routes through :class:`PgBarrier`, which arms a ``pg_barrier_state`` row+ and dispatches the header tasks onto the PG queue. A fresh instance per+ call: the barrier holds no cross-execution state, and the substrate is no+ longer selectable, so there is nothing for a module-level singleton to+ memoise.Args:batch_tasks: List of batch task signatures@@ -139,22 +109,21 @@ class WorkflowOrchestrationUtils:for the slot name constant.Returns:- Barrier handle (Celery ``AsyncResult``-shaped — ``.id``- exposed for chord-id logging) or None if no batch tasks.+ Barrier handle (``.id`` exposed for logging) or None if no+ batch tasks.Note:CRITICAL: Returns None for zero batch tasks, signaling toparent that direct pipeline status updates should behandled instead."""- return _barrier_for_transport(transport).enqueue(+ return PgBarrier().enqueue(batch_tasks,callback_task_name=callback_task_name,callback_kwargs=callback_kwargs,callback_queue=callback_queue,app_instance=app_instance,fairness=fairness,- transport=transport,)@staticmethoddiff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.pyindex 0e076ebfc..a62253d51 100644--- a/workers/shared/workflow/execution/service.py+++ b/workers/shared/workflow/execution/service.py@@ -25,7 +25,6 @@ from unstract.core.data_models import (FileOperationConstants,WorkflowDefinitionResponseData,WorkflowEndpointConfigData,- is_pg_transport,)# Import file execution tracking for proper recovery mechanism@@ -1018,11 +1017,11 @@ class WorkerWorkflowExecutionService:def _execute_structure_tool_workflow(self, execution_service: WorkflowExecutionService, file_name: str) -> None:- """Execute structure tool as Celery task instead of Docker container.+ """Execute structure tool as a task instead of a Docker container.Calls execute_structure_tool directly (same process, in-band).- Only the inner ExecutionDispatcher calls go through Celery to- the executor worker.+ Only the inner executor-RPC dispatches leave this process, and they go+ to the executor worker over the PG queue."""from file_processing.structure_tool_task import (execute_structure_tool as _execute_structure_tool,@@ -1362,36 +1361,30 @@ class WorkerWorkflowExecutionService:logger.error("No connector_id found in any configuration source")return None, {}- def _pg_destination_already_written(+ def _destination_already_written(self,*,file_hash: FileHashData,- transport: str | None,use_file_history: bool,workflow_id: str,is_api: bool,) -> bool:"""True if this file's destination write already completed in a prior run.- On the PG (at-least-once) transport a batch can be re-run after a crash- or reaper-recovery, and such a re-run bypasses discovery's FileHistory+ The queue is at-least-once, so a batch can be re-run after a crash or+ reaper-recovery, and such a re-run bypasses discovery's FileHistoryfilter — so re-check by hash (plus path for non-API) at the writeboundary. Returns True only if a COMPLETED FileHistory record alreadyexists for this file.- Scoped two ways:- - to the PG transport (the re-run duplicate this prevents is PG-specific;- always False on Celery, so that path is unchanged), and- - to ``use_file_history``: a workflow run with file history off is- contractually "rewrite every run" (mirrors the discovery FileHistory- filter), so its write is never skipped.+ Scoped to ``use_file_history``: a workflow run with file history off is+ contractually "rewrite every run" (mirrors the discovery FileHistory+ filter), so its write is never skipped.Fail-open on any lookup error (never block a legitimate write), loggingat error level so a persistent lookup regression is visible rather than asilent, permanent no-op."""- if not is_pg_transport(transport):- return Falseif not use_file_history:return Falsecache_key = file_hash.file_hash@@ -1527,16 +1520,14 @@ class WorkerWorkflowExecutionService:f"📤 File {file_hash.file_name} marked for DESTINATION processing - sending to {destination_display}")- # PG-only re-run duplicate-write guard: a re-run after a crash /+ # Re-run duplicate-write guard: a re-run after a crash /# reaper-recovery bypasses discovery's FileHistory filter, so# re-check by hash at the write boundary. If this file's# destination write already completed in a prior run, skip it,- # avoiding a duplicate destination write. A no-op on the Celery- # transport, so that path is unchanged; fail-open so a lookup+ # avoiding a duplicate destination write. Fail-open so a lookup# error never blocks a write.- if self._pg_destination_already_written(+ if self._destination_already_written(file_hash=file_hash,- transport=file_processing_context.transport,use_file_history=use_file_history,workflow_id=workflow_id,is_api=destination.is_api,diff --git a/workers/tests/executor_dispatch_fakes.py b/workers/tests/executor_dispatch_fakes.pynew file mode 100644index 000000000..09333f099--- /dev/null+++ b/workers/tests/executor_dispatch_fakes.py@@ -0,0 +1,161 @@+"""Shared fakes for executor request-reply dispatch tests.++Several operation suites (summarize, table extract, agentic, …) assert the same+two things about a tool dispatch: *which queue it lands on* and *what payload it+carries*. They used to do that against the SDK's Celery ``ExecutionDispatcher``,+mocking ``celery_app.send_task``. That dispatcher was deleted with the rest of+the Celery transport (UN-4078); the surviving path is+:class:`~unstract.workflow_execution.executor_rpc.PgExecutionDispatcher`, which+enqueues through an injected :class:`QueueTransport` instead.++This module is the single home for the fake transport those suites now share, so+the retarget did not scatter ten near-identical mock classes across the tree.+``test_executor_rpc.py`` keeps its own local ``_FakeTransport`` — it exercises+failure modes (enqueue raising, poll raising, malformed rows) that the operation+suites have no use for, and widening this one to cover them would couple the+happy-path callers to that detail.+"""++from __future__ import annotations++from typing import Any++# ``QUEUE_PREFIX`` is the queue-per-executor prefix. Named ``celery_executor_*``+# for historical reasons and NOT dead Celery surface: ``worker-pg-executor``+# subscribes to exactly these strings (chart ``workerPgExecutor``), so the names+# are live wire contract. Imported from the production constant rather than+# re-declared, so a rename cannot silently pass these tests while stranding the+# real consumer.+from unstract.workflow_execution.executor_rpc import (+ QUEUE_PREFIX,+ ExecResultRow,+ queue_for_executor,+)++__all__ = [+ "QUEUE_PREFIX",+ "EagerExecutorTransport",+ "FakeExecutorTransport",+ "callback_signature",+ "queue_for",+]+++def queue_for(executor_name: str) -> str:+ """The queue an executor's dispatches land on.++ Delegates to the production helper rather than re-deriving the rule, so an+ assertion here observes what the dispatcher actually does: a change to the+ naming fails these suites instead of them agreeing with a stale copy.+ """+ return queue_for_executor(executor_name)+++def callback_signature(task_name: str, *, queue: str = "celery_callback") -> Any:+ """A real ``Signature`` shaped the way PG self-chaining requires.++ ``signature_to_continuation`` rejects a signature with no task name, no+ queue, or any positional args. A bare ``MagicMock`` fails all three — every+ attribute it auto-creates is truthy, so it trips the positional-args guard —+ which makes it useless as a stand-in here. Imported by the suites that assert+ on the continuation payload so they cannot drift on that shape.+ """+ from celery.canvas import Signature++ return Signature(task_name, args=(), kwargs={}, queue=queue)+++class FakeExecutorTransport:+ """Records ``enqueue`` calls and replays a canned result to the poll.++ Exercises :class:`PgExecutionDispatcher` end-to-end with no database. The+ recorded ``enqueue_calls`` are plain kwarg dicts — assert on+ ``calls[0]["queue"]`` / ``["context"]`` / ``["org_id"]`` the way the old+ suites asserted on ``send_task.call_args``.+ """++ def __init__(self, *, result: dict[str, Any] | None = None) -> None:+ self.enqueue_calls: list[dict[str, Any]] = []+ self.wait_timeouts: list[float] = []+ self._result = result++ def enqueue(+ self,+ *,+ queue: str,+ context: Any,+ org_id: str,+ reply_key: str | None = None,+ on_success: Any | None = None,+ on_error: Any | None = None,+ task_id: str | None = None,+ ) -> None:+ # Mirrors the QueueTransport protocol exactly (keyword-only, fixed key+ # set) rather than swallowing **kwargs: a production signature change+ # then fails these suites instead of being silently absorbed.+ self.enqueue_calls.append(+ {+ "queue": queue,+ "context": context,+ "org_id": org_id,+ "reply_key": reply_key,+ "on_success": on_success,+ "on_error": on_error,+ "task_id": task_id,+ }+ )++ def wait_for_result(self, reply_key: str, timeout: float) -> ExecResultRow | None:+ del reply_key+ self.wait_timeouts.append(timeout)+ if self._result is None:+ return None+ return ExecResultRow(status="completed", result=self._result, error="")++ # -- convenience accessors, so callers don't index into the list inline ----++ @property+ def only_call(self) -> dict[str, Any]:+ """The single recorded enqueue; fails loudly if there wasn't exactly one."""+ assert len(self.enqueue_calls) == 1, (+ f"expected exactly one enqueue, got {len(self.enqueue_calls)}"+ )+ return self.enqueue_calls[0]++ @property+ def queue(self) -> str:+ """Queue of the single recorded enqueue."""+ return self.only_call["queue"]++ @property+ def context(self) -> Any:+ """``ExecutionContext`` of the single recorded enqueue."""+ return self.only_call["context"]+++class EagerExecutorTransport(FakeExecutorTransport):+ """Runs the ``execute_extraction`` task in-process instead of enqueueing it.++ The full-chain contract test needs dispatcher → task → orchestrator → result to+ actually execute. On the Celery transport that was done by patching+ ``send_task`` to call ``task.apply()``; the equivalent here is a transport whose+ ``enqueue`` runs the task synchronously and hands the outcome back through+ ``wait_for_result``, so the same chain is exercised through the surviving+ dispatcher.++ ``args=[context.to_dict()]`` mirrors :class:`PgClientQueueTransport.enqueue`+ exactly — if the real payload shape changes, this fake must change with it or+ the chain it claims to cover is not the one that ships.+ """++ def __init__(self, task: Any) -> None:+ super().__init__()+ self._task = task++ def enqueue(self, *, context: Any, **kwargs: Any) -> None:+ # ``context`` is named so this fake breaks loudly if the protocol drops+ # it; the rest is forwarded to the base recorder, which pins the full+ # keyword-only key set.+ super().enqueue(context=context, **kwargs)+ outcome = self._task.apply(args=[context.to_dict()])+ self._result = outcome.get()diff --git a/workers/tests/test_agentic_operations.py b/workers/tests/test_agentic_operations.pyindex 4d0fec497..713b5dc9d 100644--- a/workers/tests/test_agentic_operations.py+++ b/workers/tests/test_agentic_operations.py@@ -8,10 +8,12 @@ from unittest.mock import MagicMock, patchimport pytestfrom unstract.sdk1.execution.context import ExecutionContext, Operation-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.sdk1.execution.executor import BaseExecutorfrom unstract.sdk1.execution.registry import ExecutorRegistryfrom unstract.sdk1.execution.result import ExecutionResult+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher++from .executor_dispatch_fakes import FakeExecutorTransport, queue_forAGENTIC_OPERATIONS = ["agentic_extract",@@ -108,19 +110,17 @@ class TestAgenticExecutorRegistration:class TestAgenticQueueRouting:def test_agentic_routes_to_correct_queue(self):- queue = ExecutionDispatcher._get_queue("agentic")+ queue = queue_for("agentic")assert queue == "celery_executor_agentic"@pytest.mark.parametrize("op", AGENTIC_OPERATIONS)def test_dispatch_sends_to_agentic_queue(self, op):- mock_app = MagicMock()- mock_result = MagicMock()- mock_result.get.return_value = ExecutionResult(- success=True, data={"output": {}}- ).to_dict()- mock_app.send_task.return_value = mock_result-- dispatcher = ExecutionDispatcher(celery_app=mock_app)+ transport = FakeExecutorTransport(+ result=ExecutionResult(+ success=True, data={"output": {}}+ ).to_dict()+ )+ dispatcher = PgExecutionDispatcher(transport)ctx = ExecutionContext(executor_name="agentic",operation=op,@@ -130,9 +130,7 @@ class TestAgenticQueueRouting:)dispatcher.dispatch(ctx)- mock_app.send_task.assert_called_once()- call_kwargs = mock_app.send_task.call_args- assert call_kwargs.kwargs.get("queue") == "celery_executor_agentic"+ assert transport.queue == "celery_executor_agentic"# ---------------------------------------------------------------------------diff --git a/workers/tests/test_barrier.py b/workers/tests/test_barrier.pyindex a7c95ab5c..7bf65d182 100644--- a/workers/tests/test_barrier.py+++ b/workers/tests/test_barrier.py@@ -1,29 +1,21 @@-"""Characterisation tests for the ``Barrier`` abstraction (PG Queue Phase 6a).--The two production chord call sites-(``WorkflowOrchestrationUtils.create_chord_execution`` and-``api-deployment/tasks.py``'s inline chord) were lifted behind-``CeleryChordBarrier`` so a future Phase 6b can swap in-``RedisDecrBarrier`` (or PG-based) implementations without touching-call sites.--Three layers of coverage:--1. **Barrier protocol shape** — ``CeleryChordBarrier`` satisfies the- ``Barrier`` Protocol and ``AsyncResult`` satisfies ``BarrierHandle``.-2. **Wire equivalence with the pre-Barrier chord call** — given the- same args, ``CeleryChordBarrier.enqueue(...)`` produces the same- ``chord(header)(body)`` invocation, the same ``app.signature(...)``- args, and the same return value. A regression that bypasses the- abstraction or alters the wire surface fails one of these tests.-3. **Fairness plumbing** — when ``fairness=`` is passed, the fairness- header is attached to every enqueued task and to the callback- (closes Phase 5.1's chord-fairness gap that was deliberately- scoped out).--The single ``chord(...)`` call still lives in-``queue_backend/barrier.py`` (asserted by the inventory test in-``test_chord_sites_characterisation.py``).+"""Tests for the ``Barrier`` seam and its two production fan-out call sites.++The seam exists because ``WorkflowOrchestrationUtils.create_chord_execution`` and+``api-deployment/tasks.py`` both fan out and both need one dispatch point. It+once carried three implementations selected by ``WORKER_BARRIER_BACKEND``;+UN-4078 left only :class:`PgBarrier`.++Two layers of coverage:++1. **Protocol shape** — the surviving implementation satisfies ``Barrier``, and+ the handle satisfies ``BarrierHandle``.+2. **Call-site contracts** — the api-deployment fan-out passes API fairness,+ handles the zero-batch branch, raises rather than silently falling back on a+ falsy result with non-empty batches, and routes manual review per batch.++The wire-equivalence layer (chord call shape, AMQP fairness headers) went with+``CeleryChordBarrier``; the equivalent PG assertions live in+``test_pg_barrier.py``."""from __future__ import annotations@@ -36,17 +28,17 @@ import pytestfrom queue_backend import (Barrier,BarrierHandle,- CeleryChordBarrier,FairnessKey,)-from queue_backend.fairness import FAIRNESS_HEADER_NAME, WorkloadType+from queue_backend.fairness import WorkloadType@pytest.fixturedef app() -> MagicMock:"""Celery-app-shaped mock with a working ``.signature(...)``.- Used by every test that drives ``CeleryChordBarrier.enqueue(...)``.+ The fan-out call sites still build header signatures through a Celery app+ before handing them to the barrier, so they still need one.Shared via fixture rather than a per-class ``_make_app`` helper tokeep test classes free of boilerplate (SonarCloud S4144duplication).@@ -74,13 +66,16 @@ def mock_chord():class TestBarrierProtocolShape:- def test_celery_chord_barrier_satisfies_protocol(self):- """``CeleryChordBarrier`` is structurally a ``Barrier``.+ def test_pg_barrier_satisfies_protocol(self):+ """The one remaining implementation is structurally a ``Barrier``.- Pinned so Phase 6b's ``RedisDecrBarrier`` can be substituted at- any call site without runtime ``AttributeError``.+ Kept after ``CeleryChordBarrier`` was deleted: the Protocol still exists+ because two call sites program against it, so something must assert the+ implementation actually satisfies it."""- barrier: Barrier = CeleryChordBarrier()+ from queue_backend import PgBarrier++ barrier: Barrier = PgBarrier()assert callable(getattr(barrier, "enqueue", None))def test_barrier_handle_protocol_satisfied_by_celery_async_result(self):@@ -151,247 +146,30 @@ class TestBarrierProtocolShape:# --- Wire equivalence with the pre-Barrier chord call ----class TestCeleryChordBarrierWireEquivalence:- """``CeleryChordBarrier.enqueue(...)`` must produce the same- ``chord(batch_tasks)(callback_signature)`` invocation that the- original inline ``chord(...)`` calls produced — modulo the- additive ``x-fairness-key`` header that the barrier optionally- stamps onto signatures.- """-- def test_empty_header_returns_none_and_skips_chord(self, app, mock_chord):- """Zero-batch contract preserved: barrier returns None,- ``chord(...)`` is never called.-- Parent callers (``general/tasks.py``, ``api-deployment/tasks.py``)- rely on this signal to handle pipeline status updates directly.- """- result = CeleryChordBarrier().enqueue(- [],- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-1", "pipeline_id": "pipe-1"},- callback_queue="file_processing_callback",- app_instance=app,- )-- assert result is None- mock_chord.assert_not_called()-- def test_non_empty_header_invokes_chord_header_then_body(self, app, mock_chord):- """The original ``chord(header)(body)`` two-step call must- be preserved exactly — a refactor that flattens or reorders- these calls would change Celery's chord semantics."""- header = [MagicMock(name="h1"), MagicMock(name="h2")]-- CeleryChordBarrier().enqueue(- header,- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="file_processing_callback",- app_instance=app,- )-- # Step 1: chord(header)- mock_chord.assert_called_once_with(header)- # Step 2: chord_obj(callback_signature) — applies the chord- mock_chord.return_value.assert_called_once_with(app.signature.return_value)-- def test_callback_signature_args_match_pre_uplift_contract(self, app, mock_chord):- """``app.signature(task_name, kwargs=..., queue=...)`` is- called with exactly the args the pre-Barrier helper used."""- callback_kwargs = {- "execution_id": "exec-42",- "pipeline_id": "pipe-7",- "organization_id": "org-x",- }-- CeleryChordBarrier().enqueue(- [MagicMock(name="h")],- callback_task_name="process_batch_callback_api",- callback_kwargs=callback_kwargs,- callback_queue="api_file_processing_callback",- app_instance=app,- )-- # Without ``fairness=``, no ``headers=`` kwarg on the signature- # call — matches the pre-Barrier wire byte-for-byte.- app.signature.assert_called_once_with(- "process_batch_callback_api",- kwargs=callback_kwargs,- queue="api_file_processing_callback",- )-- def test_returns_chord_result_object(self, app, mock_chord):- """The barrier handle is whatever ``chord(header)(body)``- returns — Celery's ``AsyncResult`` in production. Callers- log ``.id`` for chord-id tracing."""- chord_result = MagicMock(name="chord_result")- mock_chord.return_value.return_value = chord_result-- result = CeleryChordBarrier().enqueue(- [MagicMock()],- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="file_processing_callback",- app_instance=app,- )-- assert result is chord_result-- def test_chord_failure_is_re_raised_after_logging(self, app, mock_chord):- """If ``chord(...)`` raises (broker outage, serialisation- error, etc.), the barrier logs and re-raises — never swallows.-- Without this, callers would treat broker failures as silent- zero-task chords and skip pipeline status updates entirely.- """- mock_chord.side_effect = RuntimeError("broker exploded")-- with pytest.raises(RuntimeError, match="broker exploded"):- CeleryChordBarrier().enqueue(- [MagicMock()],- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="file_processing_callback",- app_instance=app,- )---# --- Fairness plumbing ------class TestCeleryChordBarrierFairnessHeader:- """When ``fairness=FairnessKey(...)`` is passed, the barrier- stamps ``x-fairness-key`` on every header task and the callback.-- Closes the chord-fairness gap from Phase 5.1: bare ``dispatch()``- calls carried fairness; the two chord call sites didn't (they- bypassed ``dispatch()`` entirely). After this Phase 6a uplift,- every queue-crossing payload on the workflow-execution path now- carries the fairness slot.- """-- def test_fairness_header_stamped_on_callback_signature(self, app, mock_chord):- CeleryChordBarrier().enqueue(- [MagicMock(name="h")],- callback_task_name="process_batch_callback_api",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="api_file_processing_callback",- app_instance=app,- fairness=FairnessKey(org_id="org-1", workload_type=WorkloadType.API),- )-- headers = app.signature.call_args.kwargs.get("headers")- assert headers is not None, (- "expected ``headers=`` kwarg on app.signature when fairness "- "is passed"- )- assert headers[FAIRNESS_HEADER_NAME] == {- "org_id": "org-1",- "workload_type": "api",- "pipeline_priority": 5,- }-- def test_fairness_header_stamped_on_every_header_task(self, app, mock_chord):- """Every batch task in the header carries the fairness header- so PG Queue's per-task fairness scheduler can route each- independently.-- Header signatures are stamped via ``Signature.clone().set(...)``- (not in-place ``.set(...)``) — clone avoids cross-tenant- header leakage if a future retry path or signature cache ever- re-uses the original ``header_tasks`` list with a different- ``FairnessKey``. The test asserts ``.clone().set(headers=...)``- is called on each original task.- """- h1, h2, h3 = (MagicMock(name=f"header_task_{i}") for i in range(3))-- CeleryChordBarrier().enqueue(- [h1, h2, h3],- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="file_processing_callback",- app_instance=app,- fairness=FairnessKey(org_id="org-1", workload_type=WorkloadType.NON_API),- )-- expected = {- FAIRNESS_HEADER_NAME: {- "org_id": "org-1",- "workload_type": "non_api",- "pipeline_priority": 5,- }- }- for task in (h1, h2, h3):- # ``.clone()`` produces a fresh signature; the ``.set(...)``- # then attaches the fairness header to the clone, leaving- # the original ``task`` unchanged.- task.clone.assert_called_once_with()- task.clone.return_value.set.assert_called_once_with(headers=expected)- # Direct ``.set(...)`` on the original is NEVER called —- # the whole point of the clone-and-set pattern.- task.set.assert_not_called()-- def test_no_fairness_no_header_added(self, app, mock_chord):- """When ``fairness=None``, the barrier behaves byte-for-byte- like the pre-Barrier chord call — no ``headers=`` kwarg on- the signature, no ``.set(headers=...)`` on header tasks.-- Mixed-version rolling deploys are safe because this branch- produces the same wire as the old direct chord call.- """- header = [MagicMock(name="h1"), MagicMock(name="h2")]-- CeleryChordBarrier().enqueue(- header,- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="file_processing_callback",- app_instance=app,- )+class TestCreateChordExecutionUsesPgBarrier:+ """Pin ``WorkflowOrchestrationUtils.create_chord_execution`` to :class:`PgBarrier`.- # Callback signature: no headers kwarg.- assert "headers" not in app.signature.call_args.kwargs-- # Header tasks: ``.set(...)`` not called for headers stamping.- for task in header:- # ``set`` might be called for other reasons in production,- # but never with ``headers=`` when fairness is None.- for call in task.set.call_args_list:- assert "headers" not in call.kwargs, (- "unexpected fairness header stamped without fairness="- )---# --- Singleton routing (pins the _BARRIER dispatch point) ------class TestOrchestrationUtilsRoutesThroughSingleton:- """Pin the ``WorkflowOrchestrationUtils.create_chord_execution``- routing to the module-level ``_BARRIER`` singleton.-- A refactor that bypasses the singleton (e.g. inlining- ``CeleryChordBarrier().enqueue(...)`` inside the static method,- or going back to a direct ``chord(...)`` call) would still pass- the inventory canary in ``test_chord_sites_characterisation.py``- as long as ``chord(...)`` lives somewhere inside ``barrier.py``.- This test closes that gap so the singleton is the unambiguous- single dispatch point — ready for Phase 6b's factory swap.+ Replaces the old singleton-routing suite. There used to be a module-level+ ``_BARRIER`` chosen by ``WORKER_BARRIER_BACKEND`` plus a per-execution+ ``transport`` that decided between it and a fresh ``PgBarrier``; both went+ with the Celery transport (UN-4078). What is worth pinning now is that the+ call site still funnels through one substrate and forwards its arguments+ unchanged — a refactor that reached for a barrier directly, or dropped+ ``fairness`` on the way through, would otherwise pass silently."""- def test_create_chord_execution_delegates_to_module_barrier(self):+ def test_delegates_to_pg_barrier_with_arguments_intact(self):from shared.workflow.execution import orchestration_utilsfrom shared.workflow.execution.orchestration_utils import (WorkflowOrchestrationUtils,)app_mock = MagicMock(name="celery_app")- app_mock.signature.return_value = MagicMock(name="callback_signature")batch = [MagicMock(name="h1")]fairness = FairnessKey(org_id="org-1", workload_type=WorkloadType.API)- with patch.object(orchestration_utils, "_BARRIER") as mock_barrier:- mock_barrier.enqueue.return_value = MagicMock(name="result")+ with patch.object(orchestration_utils, "PgBarrier") as barrier_cls:+ barrier_cls.return_value.enqueue.return_value = MagicMock(name="result")WorkflowOrchestrationUtils.create_chord_execution(batch_tasks=batch,callback_task_name="process_batch_callback_api",@@ -401,54 +179,36 @@ class TestOrchestrationUtilsRoutesThroughSingleton:fairness=fairness,)- # Default transport (celery) → the env-selected singleton, with the- # per-execution transport threaded through to the substrate (9e).- mock_barrier.enqueue.assert_called_once_with(+ barrier_cls.return_value.enqueue.assert_called_once_with(batch,callback_task_name="process_batch_callback_api",callback_kwargs={"execution_id": "exec-1"},callback_queue="api_file_processing_callback",app_instance=app_mock,fairness=fairness,- transport="celery",)- def test_pg_queue_transport_bypasses_singleton_for_pgbarrier(self):- # 9e: a pg_queue execution must coordinate on Postgres regardless of- # WORKER_BARRIER_BACKEND, so it uses a fresh PgBarrier — NOT the (possibly- # chord) singleton. Pin that the singleton's enqueue is never called.- from shared.workflow.execution import orchestration_utils+ def test_no_transport_argument_is_accepted(self):+ """The per-execution ``transport`` kwarg is gone, not merely ignored.++ It was the field that let a stale payload select a Celery fan-out after+ the Celery consumers were scaled to zero. Accepting and discarding it+ would leave that call shape looking valid.+ """from shared.workflow.execution.orchestration_utils import (WorkflowOrchestrationUtils,)- app_mock = MagicMock(name="celery_app")- batch = [MagicMock(name="h1")]-- with (- patch.object(orchestration_utils, "_BARRIER") as mock_singleton,- patch.object(orchestration_utils, "PgBarrier") as mock_pgbarrier_cls,- ):+ with pytest.raises(TypeError):WorkflowOrchestrationUtils.create_chord_execution(- batch_tasks=batch,- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-pg"},- callback_queue="general",- app_instance=app_mock,- transport="pg_queue",+ batch_tasks=[MagicMock()],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-1"},+ callback_queue="q",+ app_instance=MagicMock(),+ transport="celery",)- mock_singleton.enqueue.assert_not_called() # NOT the env singleton- mock_pgbarrier_cls.assert_called_once_with() # a fresh PgBarrier- mock_pgbarrier_cls.return_value.enqueue.assert_called_once()- assert (- mock_pgbarrier_cls.return_value.enqueue.call_args.kwargs["transport"]- == "pg_queue"- )---# --- Executing tests for call-site fairness contracts ----def _load_api_deployment_tasks():"""Load ``api-deployment/tasks.py`` as a module.diff --git a/workers/tests/test_barrier_backend_selection.py b/workers/tests/test_barrier_backend_selection.pydeleted file mode 100644index 0948814e6..000000000--- a/workers/tests/test_barrier_backend_selection.py+++ /dev/null@@ -1,123 +0,0 @@-"""Tests for the ``get_barrier()`` factory + ``WORKER_BARRIER_BACKEND`` flag.--Phase 6b selects the ``Barrier`` substrate at runtime via an env flag.-This test suite pins:--- Default (env unset) → ``CeleryChordBarrier`` — protects against any- production behaviour change at merge time; downstream envs that- haven't added the new flag still run the same substrate they ran- yesterday.-- ``WORKER_BARRIER_BACKEND=chord`` → ``CeleryChordBarrier``.-- ``WORKER_BARRIER_BACKEND=redis`` → ``RedisDecrBarrier``.-- Unknown values raise ``ValueError`` — silent fallback to default- would mask a typo'd flag in production env, which the canary- must not allow.-- Case + whitespace tolerance — operators frequently set env vars- via shell scripts or k8s configmaps; ``" REDIS "`` should- resolve the same as ``"redis"``.-- ``BarrierBackend`` enum membership — the canonical string values- match what operators set in env.-"""--from __future__ import annotations--import pytest--from queue_backend import (- BarrierBackend,- CeleryChordBarrier,- PgBarrier,- RedisDecrBarrier,- get_barrier,-)---class TestBarrierBackendEnum:- def test_chord_value_is_canonical_string(self):- """The enum value is what operators set in env. If this- drifts, the factory's dispatch breaks silently for one- substrate (the one whose enum value no longer matches the- documented env value)."""- assert BarrierBackend.CHORD.value == "chord"- assert BarrierBackend.REDIS.value == "redis"- assert BarrierBackend.PG.value == "pg"-- def test_str_enum_string_equality(self):- """``StrEnum`` members compare equal to their string values —- means tests + production code can use either form- interchangeably without conversion."""- assert BarrierBackend.CHORD == "chord"- assert BarrierBackend.REDIS == "redis"- assert BarrierBackend.PG == "pg"---class TestGetBarrierFactory:- def test_default_returns_celery_chord_barrier(self, monkeypatch):- """Env unset → ``CeleryChordBarrier``. This is what protects- production from a behaviour change at merge time — downstream- envs (``unstract-cloud``, self-hosted, etc.) that haven't- added the new flag run the same substrate they ran yesterday."""- monkeypatch.delenv("WORKER_BARRIER_BACKEND", raising=False)- barrier = get_barrier()- assert isinstance(barrier, CeleryChordBarrier)-- def test_chord_selects_celery_chord_barrier(self, monkeypatch):- monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.CHORD)- assert isinstance(get_barrier(), CeleryChordBarrier)-- def test_redis_selects_redis_decr_barrier(self, monkeypatch):- monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.REDIS)- assert isinstance(get_barrier(), RedisDecrBarrier)-- def test_pg_selects_pg_barrier(self, monkeypatch):- monkeypatch.setenv("WORKER_BARRIER_BACKEND", BarrierBackend.PG)- assert isinstance(get_barrier(), PgBarrier)-- def test_unknown_value_raises_loudly(self, monkeypatch):- """Silent fallback to default would mask a typo'd production- env (``rediz``, ``REDDIS``, ``redis-decr``) — the canary- must surface the misconfiguration."""- monkeypatch.setenv("WORKER_BARRIER_BACKEND", "rediz")- with pytest.raises(ValueError, match=r"rediz"):- get_barrier()-- def test_empty_string_raises_loudly(self, monkeypatch):- """Empty string is not the same as "unset" — it's a positive- misconfiguration (e.g. an unsubstituted ``$BARRIER`` in a- shell script). Surface it rather than falling back to- default, which would mask the bug."""- monkeypatch.setenv("WORKER_BARRIER_BACKEND", "")- with pytest.raises(ValueError):- get_barrier()-- def test_case_insensitive(self, monkeypatch):- """Operators frequently set env vars via shell heredocs or- k8s configmaps with inconsistent casing. ``REDIS`` and- ``Redis`` should resolve the same as ``redis``."""- monkeypatch.setenv("WORKER_BARRIER_BACKEND", "REDIS")- assert isinstance(get_barrier(), RedisDecrBarrier)- monkeypatch.setenv("WORKER_BARRIER_BACKEND", "Chord")- assert isinstance(get_barrier(), CeleryChordBarrier)-- def test_whitespace_tolerated(self, monkeypatch):- """Stray whitespace (common from shell substitutions) is- stripped before the membership check."""- monkeypatch.setenv("WORKER_BARRIER_BACKEND", " redis ")- assert isinstance(get_barrier(), RedisDecrBarrier)-- def test_returns_fresh_instance_per_call(self, monkeypatch):- """``get_barrier()`` builds a new instance each call rather- than returning a singleton. Call sites that want a singleton- (e.g. ``orchestration_utils._BARRIER``) cache it at module- load time. Tests can flip the env per-test without leaking- a stale instance from a previous test."""- monkeypatch.setenv("WORKER_BARRIER_BACKEND", "chord")- a = get_barrier()- b = get_barrier()- assert a is not b- assert isinstance(a, CeleryChordBarrier)- assert isinstance(b, CeleryChordBarrier)---if __name__ == "__main__":- pytest.main([__file__, "-v"])diff --git a/workers/tests/test_barrier_differential.py b/workers/tests/test_barrier_differential.pydeleted file mode 100644index 99bbb1200..000000000--- a/workers/tests/test_barrier_differential.py+++ /dev/null@@ -1,232 +0,0 @@-"""Differential test — both ``Barrier`` substrates produce equivalent-callback firings for a fixed input.--This is the load-bearing test that justifies the substrate swap:-given the same header tasks + callback descriptor + fairness, both-``CeleryChordBarrier`` and ``RedisDecrBarrier`` must ultimately fire-the same callback with the same ``(task_name, args, kwargs, queue,-headers)``. The execution paths diverge — chord aggregates via-Celery's chord backend, redis aggregates via ``DECR remaining`` +-``RPUSH`` — but the observable output (what the callback receives)-is equivalent.--If this test fails, the substrate swap is *not* safe to roll out:-the callback would see different aggregated results or different-metadata between substrates, breaking executions when the flag-flips.--Note this is a unit-level differential — it pins the wire shape of-the callback dispatch, not full end-to-end equivalence. A future-integration test (against real Celery + Redis with N parallel-workers) would belong in ``test_barrier_integration.py``.-"""--from __future__ import annotations--import json-from unittest.mock import MagicMock, patch--import pytest--from queue_backend import CeleryChordBarrier, RedisDecrBarrier, barrier_decr_and_check-from queue_backend.fairness import FairnessKey, WorkloadType---# A fixed scenario both substrates run through.-_HEADER_TASK_RESULTS = [- {"batch_id": "b1", "files": ["f1", "f2"], "status": "ok"},- {"batch_id": "b2", "files": ["f3", "f4"], "status": "ok"},- {"batch_id": "b3", "files": ["f5"], "status": "ok"},-]-_CALLBACK_TASK = "process_batch_callback_api"-_CALLBACK_KWARGS = {- "execution_id": "exec-differential-1",- "pipeline_id": "pipe-x",- "organization_id": "org-y",-}-_CALLBACK_QUEUE = "api_file_processing_callback"-_FAIRNESS = FairnessKey(org_id="org-y", workload_type=WorkloadType.API)---def _capture_chord_callback_dispatch() -> dict:- """Run ``CeleryChordBarrier.enqueue`` against a mocked ``chord``- and return the callback signature's construction args.-- Chord's wire model: ``chord(header_tasks)(callback_signature)`` —- the callback_signature is built via ``app.signature(name, kwargs=...,- queue=..., headers=...)`` and then passed as the body.- """- app = MagicMock(name="app")- headers_captured = {}-- def signature_recorder(name, **kwargs):- headers_captured["task_name"] = name- headers_captured["args"] = kwargs.get("args") # None for chord callback- headers_captured["kwargs"] = kwargs.get("kwargs")- headers_captured["queue"] = kwargs.get("queue")- headers_captured["headers"] = kwargs.get("headers")- return MagicMock(name="callback_signature")-- app.signature.side_effect = signature_recorder- header = [MagicMock(name=f"h{i}") for i in range(len(_HEADER_TASK_RESULTS))]-- with patch("queue_backend.barrier.chord"):- CeleryChordBarrier().enqueue(- header,- callback_task_name=_CALLBACK_TASK,- callback_kwargs=_CALLBACK_KWARGS,- callback_queue=_CALLBACK_QUEUE,- app_instance=app,- fairness=_FAIRNESS,- )- return headers_captured---def _capture_redis_callback_dispatch() -> dict:- """Drive ``RedisDecrBarrier.enqueue`` + simulate the link task- completing for all header tasks, then capture the final- callback signature's construction args at the post-aggregation- dispatch site.- """- redis_mock = MagicMock(name="redis")- barrier = RedisDecrBarrier(redis_client=redis_mock)- header = [MagicMock(name=f"h{i}") for i in range(len(_HEADER_TASK_RESULTS))]- barrier.enqueue(- header,- callback_task_name=_CALLBACK_TASK,- callback_kwargs=_CALLBACK_KWARGS,- callback_queue=_CALLBACK_QUEUE,- app_instance=MagicMock(name="app"),- fairness=_FAIRNESS,- )-- # Now simulate each link firing. The first N-1 see remaining > 0;- # the last sees remaining == 0 and dispatches the callback.- headers_captured = {}-- def signature_recorder(name, **kwargs):- headers_captured["task_name"] = name- headers_captured["args"] = kwargs.get("args")- headers_captured["kwargs"] = kwargs.get("kwargs")- headers_captured["queue"] = kwargs.get("queue")- headers_captured["headers"] = kwargs.get("headers")- cb_sig = MagicMock(name="callback_signature")- cb_sig.apply_async.return_value = MagicMock(id="callback-task-id")- return cb_sig-- # Simulate the Lua script's return for each link invocation.- # First N-1 calls: [remaining, []]. Last call: [0, [serialised results]].- n = len(_HEADER_TASK_RESULTS)- serialised_results = [json.dumps(r) for r in _HEADER_TASK_RESULTS]- return_values = [- [n - 1 - i, []] for i in range(n - 1)- ] + [[0, serialised_results]]- script_mock = MagicMock(side_effect=return_values)-- with patch("queue_backend.redis_barrier._get_redis_client") as get_redis:- # Inside the link task, ``_get_redis_client()`` is called to- # build the script invocation. Return a mock whose- # register_script returns the script_mock that yields the- # sequenced return values.- link_redis = MagicMock(name="link_redis")- link_redis.register_script.return_value = script_mock- get_redis.return_value = link_redis-- with patch("celery.current_app") as current_app:- current_app.signature.side_effect = signature_recorder- for header_result in _HEADER_TASK_RESULTS:- barrier_decr_and_check.run(- header_result,- execution_id=_CALLBACK_KWARGS["execution_id"],- callback_descriptor={- "task_name": _CALLBACK_TASK,- "kwargs": _CALLBACK_KWARGS,- "queue": _CALLBACK_QUEUE,- "fairness_headers": _FAIRNESS.as_header(),- },- )- return headers_captured---class TestBarrierSubstrateDifferential:- """Both substrates must produce equivalent callback firings.-- A failure here means the swap is not safe — flag flipping in- Phase 6c would change the data the callback receives.- """-- def test_same_callback_task_name(self):- """Both substrates fire the same callback task name."""- chord_cap = _capture_chord_callback_dispatch()- redis_cap = _capture_redis_callback_dispatch()- assert chord_cap["task_name"] == redis_cap["task_name"] == _CALLBACK_TASK-- def test_same_callback_kwargs(self):- """Both substrates forward the same ``execution_id`` /- ``pipeline_id`` / ``organization_id`` to the callback."""- chord_cap = _capture_chord_callback_dispatch()- redis_cap = _capture_redis_callback_dispatch()- assert chord_cap["kwargs"] == redis_cap["kwargs"] == _CALLBACK_KWARGS-- def test_same_callback_queue(self):- """Both substrates route the callback to the same queue —- without this, post-Phase-6c the callback could land on a- worker that doesn't have it registered."""- chord_cap = _capture_chord_callback_dispatch()- redis_cap = _capture_redis_callback_dispatch()- assert chord_cap["queue"] == redis_cap["queue"] == _CALLBACK_QUEUE-- def test_same_fairness_headers(self):- """Both substrates ride the same ``x-fairness-key`` AMQP- header on the callback. Mismatch would mean post-flag-flip- executions route through different fairness slots, which- the L1/L2/L3 admission logic could throttle differently."""- chord_cap = _capture_chord_callback_dispatch()- redis_cap = _capture_redis_callback_dispatch()- assert chord_cap["headers"] == redis_cap["headers"] == _FAIRNESS.as_header()-- def test_redis_substrate_passes_aggregated_results_as_first_arg(self):- """Chord's callback receives ``[result_1, result_2, ...]`` via- Celery's chord-aggregator (an implicit first positional arg- Celery injects). The Redis substrate must replicate the- same shape — pass the aggregated list explicitly via- ``args=[all_results]`` on the callback signature.-- We can't observe chord's implicit arg injection from the- outside (it's the chord backend's job), but we CAN pin the- Redis substrate's behaviour: when aggregation completes,- the callback signature's ``args[0]`` MUST equal the- original header-task results list. That's the contract the- callback already depends on (``file_batch_results: list[dict]``- — see callback/tasks.py:1552)."""- redis_cap = _capture_redis_callback_dispatch()- assert redis_cap["args"] == [_HEADER_TASK_RESULTS], (- f"Redis substrate's callback args must be a list "- f"containing one list-of-results (matching chord's "- f"aggregator shape). Got: {redis_cap['args']!r}"- )-- def test_both_handle_empty_headers_identically(self):- """Empty header → ``None`` from both substrates. The caller's- ``if result is None:`` branch then handles the zero-files- contract uniformly."""- chord_result = CeleryChordBarrier().enqueue(- [],- callback_task_name=_CALLBACK_TASK,- callback_kwargs=_CALLBACK_KWARGS,- callback_queue=_CALLBACK_QUEUE,- app_instance=MagicMock(),- )- redis_result = RedisDecrBarrier(redis_client=MagicMock()).enqueue(- [],- callback_task_name=_CALLBACK_TASK,- callback_kwargs=_CALLBACK_KWARGS,- callback_queue=_CALLBACK_QUEUE,- app_instance=MagicMock(),- )- assert chord_result is None- assert redis_result is None---if __name__ == "__main__":- pytest.main([__file__, "-v"])diff --git a/workers/tests/test_chord_callback_boundary.py b/workers/tests/test_chord_callback_boundary.pyindex 055ff69cc..dc5114e8a 100644--- a/workers/tests/test_chord_callback_boundary.py+++ b/workers/tests/test_chord_callback_boundary.py@@ -21,6 +21,7 @@ from __future__ import annotationsimport jsonfrom types import SimpleNamespace+from unittest import mockfrom unittest.mock import MagicMockimport pytest@@ -767,13 +768,17 @@ class TestRealConsumerTolerance:assert aggregated["batches_processed"] == 2-class TestProcessFileBatchTransportRouting:- """9e PR 2c: ``_process_file_batch_core`` routes by ``barrier_context`` —- Celery chord path runs the stages directly (the chord ``.link`` decrements);- the PG path wraps them in ``run_batch_with_barrier`` (in-body claim + decrement).+class TestProcessFileBatchBarrierRouting:+ """``_process_file_batch_core`` routes by ``barrier_context``.++ With one present (every real fan-out injects it) the stages are wrapped in+ ``run_batch_with_barrier`` for the in-body claim + decrement. With it absent+ — now only reachable via a malformed payload, since the Celery ``.link``+ path that legitimately produced ``None`` is gone — the stages still run, but+ nothing decrements a barrier, so the call is expected to log loudly."""- def test_celery_path_runs_stages_directly(self, monkeypatch):+ def test_missing_barrier_context_runs_stages_and_logs(self, monkeypatch):from file_processing import tasks as tasks_modstages_kwargs = {}@@ -789,12 +794,15 @@ class TestProcessFileBatchTransportRouting:"run_batch_with_barrier",lambda *a, **k: barrier_calls.append(1),)- out = tasks_mod._process_file_batch_core(MagicMock(), {"fb": 1}, None)+ with mock.patch.object(tasks_mod.logger, "error") as log_error:+ out = tasks_mod._process_file_batch_core(MagicMock(), {"fb": 1}, None)assert out == {"r": "celery"}- assert barrier_calls == [] # barrier path NOT taken on the celery transport- assert stages_kwargs.get("is_pg") is False # UN-3662 guard OFF on Celery+ assert barrier_calls == [] # no barrier to wrap+ # The batch still runs, but the operator must be told the callback will+ # never fire — a silent run here would strand the execution to the reaper.+ log_error.assert_called_once()- def test_pg_path_routes_through_barrier(self, monkeypatch):+ def test_barrier_context_routes_through_barrier(self, monkeypatch):from file_processing import tasks as tasks_modstages_kwargs = {}@@ -818,7 +826,6 @@ class TestProcessFileBatchTransportRouting:assert captured["ctx"] is ctxassert out["via"] == "barrier"assert out["work"] == {"r": "work"} # work_fn runs the real stages- assert stages_kwargs.get("is_pg") is True # UN-3662 guard ON on PGif __name__ == "__main__":diff --git a/workers/tests/test_chord_sites_characterisation.py b/workers/tests/test_chord_sites_characterisation.pyindex 5732ef44e..1b683140c 100644--- a/workers/tests/test_chord_sites_characterisation.py+++ b/workers/tests/test_chord_sites_characterisation.py@@ -1,365 +1,110 @@-"""Characterisation tests for the chord-callback boundary, now lifted-behind the ``Barrier`` abstraction (PG Queue Phase 6a).--Originally written to lock down the inline ``celery.chord(...)`` calls-at two call sites before they were lifted. Post-uplift the inline-calls are gone — both call sites delegate to-``WorkflowOrchestrationUtils.create_chord_execution`` which in turn-delegates to ``CeleryChordBarrier.enqueue(...)``. The single remaining-``chord(...)`` call lives inside ``workers/queue_backend/barrier.py``.--Sites covered (assertions unchanged from the original pre-uplift-suite — they characterise the **same** ``chord(header)(body)``-contract, now reached one indirection later):-1. ``shared/workflow/execution/orchestration_utils.py`` —- ``WorkflowOrchestrationUtils.create_chord_execution`` (centralised helper).-2. ``api-deployment/tasks.py`` — same helper, called inline inside- ``_run_workflow_api`` (post-uplift it no longer calls ``chord(...)``- directly; covered by the inventory test below).--Chord is the highest-risk Celery construct called out in the PG Queue-decision doc (silent task drops at ~130K-task scale). A future-Phase 6b will introduce ``RedisDecrBarrier`` as a second implementation-behind a flag; these tests stay green through that transition because-they characterise the **Celery chord** code path specifically.+"""Inventory canary: no Celery ``chord(...)`` anywhere in ``workers/``.++Chord was the fan-out primitive before the PG barrier, and the highest-risk+Celery construct at our scale (silent task drops at ~130K tasks). This suite+originally characterised the two inline ``chord(header)(body)`` call sites, then+narrowed to "exactly one call site, inside ``queue_backend/barrier.py``" once+they were lifted behind the ``Barrier`` abstraction.++UN-4078 deleted that last call site along with ``CeleryChordBarrier``, so the+expected count is now **zero**. What is left is worth keeping rather than+deleting with the rest: chord is still importable from the Celery library that+the PG consumer itself is built on, so nothing at the type or import level stops+someone reintroducing one. A reintroduced chord would publish to a broker with+no consumers, and — unlike a bad PG dispatch — would fail silently, with no+``pg_barrier_state`` row for the reaper to recover from.++The behavioural characterisation tests that used to live here were deleted with+the code they mocked: they patched ``queue_backend.barrier.chord``, which no+longer exists. The fan-out contract they covered (zero batches → ``None``,+non-empty → enqueue, failures re-raise) is covered against the surviving+substrate in ``test_pg_barrier.py`` and ``test_barrier.py``."""-from unittest.mock import MagicMock, patch+import pathlib+import reimport pytest+# Top-level directories under workers/ that are not production source.+_SKIP_TOP_DIRS = {"tests", "__pycache__", "htmlcov", ".venv"}-# --- Site 1: WorkflowOrchestrationUtils.create_chord_execution ------class TestCreateChordExecution:- """Characterise the centralised chord helper.-- This is the easily-testable site. PR #13 should route both this helper- AND the inline call (Site 2) through the new Barrier abstraction.- """-- def _make_app_instance(self):- """Build a Celery app-shaped mock with a working ``.signature(...)``."""- app = MagicMock()- # signature() returns an opaque "signature" object — represent it as a- # MagicMock that we can assert on by identity later.- app.signature.return_value = MagicMock(name="callback_signature")- return app-- def test_empty_batch_tasks_returns_none_and_skips_chord(self):- """Zero files: helper short-circuits with None and never calls chord()."""- from shared.workflow.execution.orchestration_utils import (- WorkflowOrchestrationUtils,- )-- app = self._make_app_instance()-- with patch("queue_backend.barrier.chord") as mock_chord:- result = WorkflowOrchestrationUtils.create_chord_execution(- batch_tasks=[],- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-1", "pipeline_id": "pipe-1"},- callback_queue="file_processing_callback",- app_instance=app,- )-- assert result is None- mock_chord.assert_not_called()-- def test_non_empty_batch_tasks_invokes_chord(self):- """Non-empty batch: chord(batch_tasks)(callback_signature) is called."""- from shared.workflow.execution.orchestration_utils import (- WorkflowOrchestrationUtils,- )-- app = self._make_app_instance()- batch_tasks = [MagicMock(name="batch_task_1"), MagicMock(name="batch_task_2")]-- with patch("queue_backend.barrier.chord") as mock_chord:- # chord(batch_tasks) returns a chord object; calling it with the- # callback signature returns the chord result. Both calls must- # happen for this characterisation.- chord_obj = MagicMock(name="chord_object")- mock_chord.return_value = chord_obj-- WorkflowOrchestrationUtils.create_chord_execution(- batch_tasks=batch_tasks,- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="file_processing_callback",- app_instance=app,- )-- # First call: chord(batch_tasks)- mock_chord.assert_called_once_with(batch_tasks)- # Second call: chord_obj(callback_signature) — applies the chord- chord_obj.assert_called_once_with(app.signature.return_value)-- def test_callback_signature_is_built_with_correct_kwargs(self):- """The callback signature must be created with the exact task name,- kwargs, and queue passed in. PR #13 must preserve these."""- from shared.workflow.execution.orchestration_utils import (- WorkflowOrchestrationUtils,- )-- app = self._make_app_instance()- batch_tasks = [MagicMock(name="batch")]- callback_kwargs = {- "execution_id": "exec-42",- "pipeline_id": "pipe-7",- "organization_id": "org-x",- }-- with patch("queue_backend.barrier.chord") as mock_chord:- mock_chord.return_value = MagicMock()- WorkflowOrchestrationUtils.create_chord_execution(- batch_tasks=batch_tasks,- callback_task_name="process_batch_callback_api",- callback_kwargs=callback_kwargs,- callback_queue="api_file_processing_callback",- app_instance=app,- )-- app.signature.assert_called_once_with(- "process_batch_callback_api",- kwargs=callback_kwargs,- queue="api_file_processing_callback",- )-- def test_returns_chord_result_object(self):- """The helper must return whatever ``chord(...)(...)`` returns —- callers depend on this return value to track the chord."""- from shared.workflow.execution.orchestration_utils import (- WorkflowOrchestrationUtils,- )-- app = self._make_app_instance()-- with patch("queue_backend.barrier.chord") as mock_chord:- chord_obj = MagicMock()- chord_result = MagicMock(name="chord_result_object")- chord_obj.return_value = chord_result- mock_chord.return_value = chord_obj-- result = WorkflowOrchestrationUtils.create_chord_execution(- batch_tasks=[MagicMock()],- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="file_processing_callback",- app_instance=app,- )-- assert result is chord_result-- def test_chord_failure_is_re_raised_after_logging(self):- """If chord() raises, the helper logs and re-raises (not swallowed)."""- from shared.workflow.execution.orchestration_utils import (- WorkflowOrchestrationUtils,- )-- app = self._make_app_instance()-- with patch("queue_backend.barrier.chord") as mock_chord:- mock_chord.side_effect = RuntimeError("broker exploded")- with pytest.raises(RuntimeError, match="broker exploded"):- WorkflowOrchestrationUtils.create_chord_execution(- batch_tasks=[MagicMock()],- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="file_processing_callback",- app_instance=app,- )+def _production_py_files() -> list[pathlib.Path]:+ workers_root = pathlib.Path(__file__).parent.parent+ out = []+ for py in workers_root.rglob("*.py"):+ rel_parts = py.relative_to(workers_root).parts+ if rel_parts and rel_parts[0] in _SKIP_TOP_DIRS:+ continue+ out.append(py)+ return out-# --- Site 1 (mixin wrapper): WorkflowOrchestrationMixin.create_chord ----+def _scan(pattern: re.Pattern, *, skip_comments: bool) -> list[str]:+ workers_root = pathlib.Path(__file__).parent.parent+ hits = []+ for py in _production_py_files():+ for line_no, line in enumerate(py.read_text().splitlines(), start=1):+ if skip_comments and line.lstrip().startswith("#"):+ continue+ if pattern.search(line):+ hits.append(f"{py.relative_to(workers_root)}:{line_no}")+ return hits-class TestWorkflowOrchestrationMixinCreateChord:- """Characterise the mixin wrapper around ``create_chord_execution``.- The mixin adds two unique behaviours over the static helper:- 1. Extracts ``self.app`` from the task context.- 2. Raises ``RuntimeError`` when no app is bound to the task.+class TestNoChordRemains:+ """``chord(...)`` must not reappear in production worker code."""- Both must be preserved by PR #13 when it lifts the mixin to use Barrier.- """+ def test_no_chord_invocation(self):+ """No assignment-form ``= chord(`` call anywhere in workers/.- def test_create_chord_extracts_app_from_self_and_delegates(self):- """The mixin must read ``self.app`` and forward it to the static helper."""- from shared.workflow.execution.orchestration_utils import (- WorkflowOrchestrationMixin,- WorkflowOrchestrationUtils,+ **Known blind spots**, unchanged from when this asserted a count of one:+ a return-value-discarded ``chord(batch)(cb)`` evades the assignment-form+ regex, and an aliased import (``from celery import chord as c``) or+ module-attribute access (``import celery`` + ``celery.chord(...)``)+ evades both this and the import canary below. Those shapes are+ non-idiomatic and appear nowhere in the tree; broaden these canaries if+ one ever ships.+ """+ hits = _scan(re.compile(r"=\s*chord\("), skip_comments=True)+ assert hits == [], (+ "Celery chord(...) call site(s) found in workers/:\n "+ + "\n ".join(hits)+ + "\nChord was removed with the Celery transport (UN-4078). A chord "+ "publishes to a broker that has no consumers, and fails silently — "+ "no pg_barrier_state row is written, so the reaper cannot recover "+ "the stranded execution. Fan out through "+ "WorkflowOrchestrationUtils.create_chord_execution (PgBarrier).")- # Build a synthetic task-like object carrying an `app` attribute.- task = type("FakeTask", (WorkflowOrchestrationMixin,), {})()- task.app = MagicMock(name="celery_app")- task.app.signature.return_value = MagicMock(name="callback_signature")-- with patch.object(- WorkflowOrchestrationUtils, "create_chord_execution"- ) as mock_static:- mock_static.return_value = MagicMock(name="chord_result")- batch = [MagicMock()]- kwargs = {"execution_id": "exec-mixin"}- task.create_chord(- batch_tasks=batch,- callback_task_name="process_batch_callback",- callback_kwargs=kwargs,- callback_queue="file_processing_callback",- )-- # Mixin now forwards ``fairness=`` (None when caller omits it)- # to ``create_chord_execution``. The positional args stay the- # same.- mock_static.assert_called_once_with(- batch,- "process_batch_callback",- kwargs,- "file_processing_callback",- task.app,- fairness=None,+ def test_no_chord_import(self):+ """``from celery import chord`` must not appear in workers/."""+ hits = _scan(+ re.compile(r"^\s*from\s+celery\s+import\s+.*\bchord\b"), skip_comments=False)-- def test_create_chord_raises_when_no_app_bound(self):- """No ``self.app`` (e.g., called outside a Celery task context):- the mixin must raise ``RuntimeError`` rather than silently failing- or passing ``None`` downstream to chord()."""- from shared.workflow.execution.orchestration_utils import (- WorkflowOrchestrationMixin,+ assert hits == [], (+ "`from celery import chord` found in workers/:\n " + "\n ".join(hits))- task = type("FakeTask", (WorkflowOrchestrationMixin,), {})()- # Deliberately do NOT set task.app — leave it absent.-- with pytest.raises(RuntimeError, match="Celery app instance not available"):- task.create_chord(- batch_tasks=[MagicMock()],- callback_task_name="process_batch_callback",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="file_processing_callback",- )---# --- Cross-site invariant: chord call-site inventory ------class TestChordSiteInventory:- """Post-Barrier-uplift invariant: exactly ONE ``chord(...)`` call- site, inside ``queue_backend/barrier.py``.-- Before the uplift there were two inline ``chord(...)`` calls (in- ``orchestration_utils.py`` and ``api-deployment/tasks.py``). The- refactor lifted both behind ``CeleryChordBarrier`` so the chord- primitive has a single home — a future ``RedisDecrBarrier`` swap- can target one location instead of chasing call sites.-- If a third ``chord(...)`` call appears anywhere outside- ``barrier.py``, this test fails — preventing a silent regression- where a future PR bypasses the abstraction.- """-- _CHORD_HOST = "queue_backend/barrier.py"-- def test_chord_invocation_lives_only_in_barrier(self):- """Count chord(...) invocations (not imports) in workers/ source.-- Tightened regex (after the original ``(?:^|[\\s=(])chord\\(``- could match docstring prose mentioning ``chord(...)`` after- whitespace): now requires assignment form ``= chord(`` (with- optional whitespace), which matches the production call at- ``barrier.py``'s ``result = chord(header_tasks)(callback_signature)``- but rejects docstring / comment mentions like ``the chord(...)- primitive``.+ def test_scanner_sees_a_realistic_tree(self):+ """Non-vacuity lock: the scan must actually walk production files.- **Known blind spot**: a return-value-discarded ``chord(batch)(cb)``- at column 0 (no assignment) would slip past this regex AND the- comment-line filter. The companion ``test_chord_import_only_in_barrier``- canary catches the common ``from celery import chord`` import- form — so a stray call site reachable via that import would- still trip it. However, aliased imports (``from celery import- chord as c``) or module-attribute access (``import celery`` +- ``celery.chord(...)``) would evade *both* canaries. We accept- this gap because (a) those import shapes are non-idiomatic and- not used anywhere in workers/, and (b) ``barrier.py`` is the- sole sanctioned chord call site in the codebase, so any stray- call would surface in code review long before runtime. If a- future PR ships either of those import shapes outside- ``barrier.py``, the canaries need broadening.+ Without this, a path bug (wrong root, over-broad skip list) would make+ both canaries above pass over an empty set and report all-clear forever+ — the exact failure mode a zero-expectation assertion invites."""- import pathlib- import re-- workers_root = pathlib.Path(__file__).parent.parent- # Anchor to the top-level directory relative to workers_root so we- # don't accidentally exclude legitimately-named subdirectories like- # `workers/shared/tests_helpers/`.- skip_top_dirs = {"tests", "__pycache__", "htmlcov", ".venv"}-- # Assignment-form ``= chord(`` — production call sites all- # capture the AsyncResult into a variable. Docstring prose- # never uses this form.+ files = _production_py_files()+ assert len(files) > 50, f"expected a populated tree, walked {len(files)} files"+ names = {f.name for f in files}+ assert "pg_barrier.py" in names+ assert "barrier.py" in names++ def test_detector_matches_a_known_bad_line(self):+ """Positive-detection lock for the invocation regex."""pattern = re.compile(r"=\s*chord\(")-- hits = []- for py in workers_root.rglob("*.py"):- rel_parts = py.relative_to(workers_root).parts- if rel_parts and rel_parts[0] in skip_top_dirs:- continue- text = py.read_text()- for line_no, line in enumerate(text.splitlines(), start=1):- stripped = line.lstrip()- # Skip pure comment lines (belt-and-suspenders — the- # regex already rejects most of these via the ``=``- # prefix requirement, but a comment like ``# x = chord(...)``- # would otherwise sneak through).- if stripped.startswith("#"):- continue- if pattern.search(line):- hits.append(f"{py.relative_to(workers_root)}:{line_no}")-- assert len(hits) == 1, (- "Expected exactly 1 chord(...) call site in workers/ (inside "- f"{self._CHORD_HOST}), found {len(hits)}:\n "- + "\n ".join(hits)- + "\nPhase 6a lifted both call sites behind CeleryChordBarrier — "- "a new direct chord(...) call bypasses the Barrier abstraction "- "and breaks the future Phase 6b RedisDecrBarrier swap point."- )- assert self._CHORD_HOST in hits[0], (- f"chord(...) call must live in {self._CHORD_HOST}, found at {hits[0]}"- )-- def test_chord_import_only_in_barrier(self):- """`from celery import chord` should appear in exactly one file —- ``queue_backend/barrier.py`` — post-Barrier uplift."""- import pathlib- import re-- workers_root = pathlib.Path(__file__).parent.parent- skip_top_dirs = {"tests", "__pycache__", "htmlcov", ".venv"}-- pattern = re.compile(r"^\s*from\s+celery\s+import\s+chord\b")-- hits = []- for py in workers_root.rglob("*.py"):- rel_parts = py.relative_to(workers_root).parts- if rel_parts and rel_parts[0] in skip_top_dirs:- continue- text = py.read_text()- for line_no, line in enumerate(text.splitlines(), start=1):- if pattern.search(line):- hits.append(f"{py.relative_to(workers_root)}:{line_no}")-- assert len(hits) == 1, (- f"Expected `from celery import chord` in exactly 1 file, found "- f"{len(hits)}:\n " + "\n ".join(hits)- )- assert self._CHORD_HOST in hits[0], (- f"chord import must live in {self._CHORD_HOST}, found at {hits[0]}"- )+ assert pattern.search(" result = chord(header_tasks)(callback)")+ assert not pattern.search(" # the chord(...) primitive is gone")if __name__ == "__main__":diff --git a/workers/tests/test_dispatch_sites_characterisation.py b/workers/tests/test_dispatch_sites_characterisation.pyindex 9bf4daad3..e1573c08b 100644--- a/workers/tests/test_dispatch_sites_characterisation.py+++ b/workers/tests/test_dispatch_sites_characterisation.py@@ -16,6 +16,7 @@ drops a field, rewires a queue, or changes an endpoint fails loudly:from unittest.mock import MagicMock, patchimport pytest+from queue_backend import QueueBackendfrom shared.patterns.notification.helper import (ENQUEUE_BUFFER_ENDPOINT,_enqueue_to_buffer,@@ -219,11 +220,13 @@ class TestSchedulerDispatchSite:assert args[4] is True # scheduled flag (always True here)def test_dispatch_kwargs_layout(self):- """Kwargs MUST contain use_file_history, pipeline_id, and transport.+ """Kwargs MUST contain exactly use_file_history, pipeline_id and the+ write-only ``transport`` shim.- ``transport`` (9e) is carried in the task payload so the pipeline stays- on one transport end-to-end. It defaults to ``"celery"`` when the- create-execution response omits it (older backend / inert PR 1).+ Nothing reads ``transport`` after UN-4078, but a pre-UN-4078 general+ worker falls back to Celery when it is absent, so it stays written for+ one release. Asserting the whole dict is deliberate: it fails on a stray+ key, and it fails if the shim is dropped too early."""from scheduler.tasks import _execute_scheduled_workflow@@ -237,50 +240,43 @@ class TestSchedulerDispatchSite:assert kwargs == {"use_file_history": True,"pipeline_id": "pipe-007",- "transport": "celery",- }-- def test_dispatch_carries_backend_resolved_transport(self):- """The transport the backend returns from create-execution is threaded- verbatim into the dispatched task's payload (payload-carry, 9e)."""- from scheduler.tasks import _execute_scheduled_workflow-- api = MagicMock()- api.create_workflow_execution.return_value = {- "execution_id": "exec-123","transport": "pg_queue",}- with patch("scheduler.tasks.dispatch") as mock_dispatch:- _execute_scheduled_workflow(api, self._make_context())-- assert mock_dispatch.call_args.kwargs["kwargs"]["transport"] == "pg_queue"+ def test_dispatch_ignores_a_transport_field_from_the_backend(self):+ """A ``transport`` value from the backend response is never forwarded.- def test_pg_transport_routes_dispatch_to_pg_backend(self):- """The one line that actually routes the scheduled orchestrator onto PG:- transport=="pg_queue" → dispatch(backend=QueueBackend.PG) (identity, not- the allow-list)."""- from queue_backend import QueueBackend+ The payload carries the constant ``pg_queue`` shim regardless of what the+ create-execution response says, so a stale ``"celery"`` cannot reach an+ un-upgraded consumer that still branches on it.+ """from scheduler.tasks import _execute_scheduled_workflowapi = MagicMock()api.create_workflow_execution.return_value = {"execution_id": "exec-123",- "transport": "pg_queue",+ "transport": "celery",}+with patch("scheduler.tasks.dispatch") as mock_dispatch:_execute_scheduled_workflow(api, self._make_context())+ assert mock_dispatch.call_args.kwargs["kwargs"]["transport"] == "pg_queue"assert mock_dispatch.call_args.kwargs["backend"] is QueueBackend.PG- def test_celery_transport_leaves_backend_none(self):- """transport=="celery" → backend=None (legacy Celery dispatch unchanged)."""+ def test_backend_is_always_pg(self):+ """The scheduled orchestrator always pins the dispatch to PG.++ This used to be conditional on the payload's transport, with ``None``+ (defer to select_backend) on the Celery branch. Stating it explicitly+ keeps the whole execution pipeline pinned at its entry point.+ """from scheduler.tasks import _execute_scheduled_workflowwith patch("scheduler.tasks.dispatch") as mock_dispatch:_execute_scheduled_workflow(self._make_api_client(), self._make_context())- assert mock_dispatch.call_args.kwargs["backend"] is None+ assert mock_dispatch.call_args.kwargs["backend"] is QueueBackend.PGdef test_no_dispatch_when_execution_creation_fails(self):"""If api_client.create_workflow_execution returns no execution_id,diff --git a/workers/tests/test_dispatch_with_callback.py b/workers/tests/test_dispatch_with_callback.pyindex dce8b73f9..f183e68bd 100644--- a/workers/tests/test_dispatch_with_callback.py+++ b/workers/tests/test_dispatch_with_callback.py@@ -16,9 +16,11 @@ from executor.executors.constants import (PromptServiceConstants as PSKeys,)from unstract.sdk1.execution.context import ExecutionContext, Operation-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.sdk1.execution.registry import ExecutorRegistryfrom unstract.sdk1.execution.result import ExecutionResult+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher++from .executor_dispatch_fakes import FakeExecutorTransport, callback_signature# ---------------------------------------------------------------------------# Patch targets@@ -176,40 +178,43 @@ def _make_output(name="field_a", prompt="What is the revenue?", **overrides):class TestDispatchWithCallback:- """Verify dispatch_with_callback passes link/link_error to send_task."""+ """Callbacks ride the enqueue payload as continuations, not Celery links.++ PG has no broker mechanism to fire ``link`` / ``link_error``, so the executor+ consumer self-chains instead: the signatures are translated to serialisable+ ``ContinuationSpec``s and carried in the message. These assertions therefore+ check the *payload*, where the old suite checked ``send_task`` kwargs.+ """- def test_callback_kwargs_passed(self):- mock_app = MagicMock()- mock_app.send_task.return_value = MagicMock(id="task-123")- dispatcher = ExecutionDispatcher(celery_app=mock_app)+ def _dispatcher(self):+ transport = FakeExecutorTransport(result={"success": True, "data": {}})+ return PgExecutionDispatcher(transport), transport+ def test_callback_continuations_and_task_id_passed(self):+ dispatcher, transport = self._dispatcher()ctx = ExecutionContext(executor_name="legacy",operation="answer_prompt",run_id="run-cb-1",execution_source="ide",)- on_success = MagicMock(name="success_sig")- on_error = MagicMock(name="error_sig")-result = dispatcher.dispatch_with_callback(ctx,- on_success=on_success,- on_error=on_error,+ on_success=callback_signature("cb.success"),+ on_error=callback_signature("cb.error"),task_id="pre-generated-id",)- call_kwargs = mock_app.send_task.call_args- assert call_kwargs.kwargs["link"] is on_success- assert call_kwargs.kwargs["link_error"] is on_error- assert call_kwargs.kwargs["task_id"] == "pre-generated-id"- assert result.id == "task-123"-- def test_no_callbacks_omits_link_kwargs(self):- mock_app = MagicMock()- mock_app.send_task.return_value = MagicMock(id="task-456")- dispatcher = ExecutionDispatcher(celery_app=mock_app)+ call = transport.only_call+ assert call["on_success"]["task_name"] == "cb.success"+ assert call["on_error"]["task_name"] == "cb.error"+ assert call["task_id"] == "pre-generated-id"+ # The handle echoes the caller's task_id — on Celery this came back off+ # the AsyncResult, so a caller reading ``.id`` is unaffected by the move.+ assert result.id == "pre-generated-id"+ def test_no_callbacks_leaves_continuations_unset(self):+ dispatcher, transport = self._dispatcher()ctx = ExecutionContext(executor_name="legacy",operation="extract",@@ -218,25 +223,13 @@ class TestDispatchWithCallback:)dispatcher.dispatch_with_callback(ctx)- call_kwargs = mock_app.send_task.call_args- assert "link" not in call_kwargs.kwargs- assert "link_error" not in call_kwargs.kwargs-- def test_no_app_raises(self):- dispatcher = ExecutionDispatcher(celery_app=None)- ctx = ExecutionContext(- executor_name="legacy",- operation="extract",- run_id="run-cb-3",- execution_source="tool",- )- with pytest.raises(ValueError, match="No Celery app"):- dispatcher.dispatch_with_callback(ctx)+ call = transport.only_call+ assert call["on_success"] is None+ assert call["on_error"] is None--# ----------------------------------------------------------------------------# 5C: ide_index compound operation through eager chain-# ---------------------------------------------------------------------------+ # ``test_no_app_raises`` is gone with the Celery dispatcher: PG dispatch is+ # transport-injected, so there is no "no app configured" state to raise on.+ # The enqueue-failure contract it stood in for lives in test_executor_rpc.py.class TestIdeIndexEagerChain:@@ -962,18 +955,14 @@ class TestOperationEnum:class TestDispatcherModes:- """Verify all three dispatch modes work."""+ """Verify all three dispatch modes work on the PG transport."""def test_dispatch_sync(self):- """dispatch() calls send_task and .get()."""- mock_app = MagicMock()- async_result = MagicMock()- async_result.get.return_value = ExecutionResult(- success=True, data={"test": 1}- ).to_dict()- mock_app.send_task.return_value = async_result-- dispatcher = ExecutionDispatcher(celery_app=mock_app)+ """dispatch() enqueues once and blocks for the result row."""+ transport = FakeExecutorTransport(+ result=ExecutionResult(success=True, data={"test": 1}).to_dict()+ )+ dispatcher = PgExecutionDispatcher(transport)ctx = ExecutionContext(executor_name="legacy",operation="extract",@@ -983,15 +972,13 @@ class TestDispatcherModes:result = dispatcher.dispatch(ctx, timeout=10)assert result.success- mock_app.send_task.assert_called_once()- async_result.get.assert_called_once()+ assert len(transport.enqueue_calls) == 1+ assert transport.wait_timeouts == [10]def test_dispatch_async(self):- """dispatch_async() returns task_id without blocking."""- mock_app = MagicMock()- mock_app.send_task.return_value = MagicMock(id="async-id")-- dispatcher = ExecutionDispatcher(celery_app=mock_app)+ """dispatch_async() returns the task id it enqueued, without blocking."""+ transport = FakeExecutorTransport()+ dispatcher = PgExecutionDispatcher(transport)ctx = ExecutionContext(executor_name="legacy",operation="extract",@@ -1000,5 +987,8 @@ class TestDispatcherModes:)task_id = dispatcher.dispatch_async(ctx)- assert task_id == "async-id"- mock_app.send_task.assert_called_once()+ # PG mints the id itself (the Celery path read it off the AsyncResult),+ # so assert it round-trips onto the message rather than pinning a literal.+ assert task_id+ assert transport.only_call["task_id"] == task_id+ assert transport.wait_timeouts == []diff --git a/workers/tests/test_executor_dispatch.py b/workers/tests/test_executor_dispatch.pyindex e57e329db..385dae472 100644--- a/workers/tests/test_executor_dispatch.py+++ b/workers/tests/test_executor_dispatch.py@@ -1,22 +1,20 @@-"""Tests for the file_processing → executor boundary: header forwarding-through ExecutionDispatcher and inventory canary for raw send_task calls.+"""Tests for the file_processing → executor boundary: payload/queue routing+through the PG executor dispatcher, and an inventory canary for raw send_task calls."""from __future__ import annotationsimport astfrom typing import Any-from unittest.mock import MagicMockimport pytest-from celery import Celery-from queue_backend import FairnessKeyfrom queue_backend.fairness import WorkloadTypefrom unstract.sdk1.execution.context import ExecutionContext, Operation-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcherfrom .canary_helpers import iter_production_trees+from .executor_dispatch_fakes import FakeExecutorTransport, callback_signature# Promote ``UserWarning`` from ``iter_production_trees`` (emitted on# unparseable production files) to a test failure. Without this an@@ -43,85 +41,71 @@ def _make_context(**overrides: Any) -> ExecutionContext:return ExecutionContext(**defaults)-class TestExecutionDispatcherForwardsHeaders:- """``ExecutionDispatcher`` propagates the ``headers`` kwarg to Celery."""+class TestPgDispatchCarriesOrgInPayload:+ """Org routing rides the enqueue payload, and ``headers=`` is rejected.++ This replaces the old ``ExecutionDispatcher`` header-forwarding suite. That+ contract is deliberately gone, not merely relocated: the PG dispatch methods+ take **no** ``headers`` argument and carry org/routing in the payload instead+ (``transport.enqueue(..., org_id=...)``). Locking the absence matters —+ three call sites kept passing ``headers=`` after the routing dispatcher was+ removed in UN-4046 and every extraction raised ``TypeError`` until they were+ found. A test that only checked the happy path would not have caught it.+ """- def _patched_app(self) -> tuple[ExecutionDispatcher, MagicMock]:- app = MagicMock(spec=Celery)- async_result = MagicMock()- async_result.id = "task-1"- async_result.get.return_value = {"success": True, "data": {}}- app.send_task.return_value = async_result- return ExecutionDispatcher(celery_app=app), app+ def _dispatcher(self) -> tuple[PgExecutionDispatcher, FakeExecutorTransport]:+ transport = FakeExecutorTransport(result={"success": True, "data": {}})+ return PgExecutionDispatcher(transport), transport- # The three dispatch entry points share the same header-forwarding- # contract via ``_build_send_kwargs``; parametrize over them so a- # divergence (e.g. one method dropping ``headers=``) surfaces with- # per-method failure granularity via the parametrize IDs.@pytest.mark.parametrize("method", ["dispatch", "dispatch_async", "dispatch_with_callback"])- def test_forwards_headers(self, method):- # Use ``FairnessKey.as_header()`` as the fixture rather than- # hand-built dicts so the test exercises the exact wire shape- # real producers emit (including ``pipeline_priority``).- headers = FairnessKey(- org_id="org-1", workload_type=WorkloadType.NON_API- ).as_header()- d, app = self._patched_app()- getattr(d, method)(_make_context(), headers=headers)- assert app.send_task.call_args.kwargs["headers"] == headers+ def test_org_id_travels_in_the_payload(self, method):+ d, transport = self._dispatcher()+ getattr(d, method)(_make_context(organization_id="org-1"))+ assert transport.only_call["org_id"] == "org-1"@pytest.mark.parametrize("method", ["dispatch", "dispatch_async", "dispatch_with_callback"])- def test_omits_headers_when_none(self, method):- # Caller passes no headers ⇒ ``headers`` kwarg not forwarded- # to send_task (preserves the call shape Celery's link /- # link_error handling expects).- d, app = self._patched_app()- getattr(d, method)(_make_context())- assert "headers" not in app.send_task.call_args.kwargs+ def test_no_headers_kwarg_accepted(self, method):+ """Passing ``headers=`` must fail loudly, not be silently absorbed."""+ d, _ = self._dispatcher()+ with pytest.raises(TypeError):+ getattr(d, method)(_make_context(), headers={"x-fairness-key": {}})@pytest.mark.parametrize("method", ["dispatch", "dispatch_async", "dispatch_with_callback"])- def test_omits_headers_when_empty_dict(self, method):- """Empty header dicts are dropped (treated as a no-headers- call). ``FairnessKey.as_header()`` can never legitimately- return ``{}`` — forwarding it would document a producer-side- build bug rather than catch it.- """- d, app = self._patched_app()- getattr(d, method)(_make_context(), headers={})- assert "headers" not in app.send_task.call_args.kwargs-- def test_dispatch_with_callback_combines_headers_and_callbacks(self):- """All four optional kwargs (headers, on_success, on_error,- task_id) land on the same ``send_task`` call. A merge bug in- ``_build_send_kwargs`` would slip through the single-kwarg- forwarding tests above.+ def test_queue_derives_from_executor_name(self, method):+ d, transport = self._dispatcher()+ getattr(d, method)(_make_context(executor_name="table"))+ assert transport.queue == "celery_executor_table"++ def test_dispatch_with_callback_carries_continuations_and_task_id(self):+ """Callbacks ride the payload as continuations, not Celery link kwargs.++ The signatures are built with a real name/queue and no positional args+ because ``signature_to_continuation`` rejects all three otherwise — PG+ self-chaining routes by the row's queue and supports kwargs-only+ callbacks. A bare ``MagicMock`` passes none of those checks, so shaping+ them here is what makes the assertion meaningful."""- from celery.canvas import Signature-- d, app = self._patched_app()- headers = FairnessKey(- org_id="org-1", workload_type=WorkloadType.NON_API- ).as_header()- on_success = MagicMock(spec=Signature)- on_error = MagicMock(spec=Signature)- d.dispatch_with_callback(+ d, transport = self._dispatcher()+ on_success = callback_signature("cb.success", queue="celery_callback")+ on_error = callback_signature("cb.error", queue="celery_callback")+ handle = d.dispatch_with_callback(_make_context(),on_success=on_success,on_error=on_error,task_id="t-1",- headers=headers,)- kwargs = app.send_task.call_args.kwargs- assert kwargs["headers"] == headers- assert kwargs["link"] is on_success- assert kwargs["link_error"] is on_error- assert kwargs["task_id"] == "t-1"+ call = transport.only_call+ assert call["task_id"] == "t-1"+ assert handle.id == "t-1"+ assert call["on_success"]["task_name"] == "cb.success"+ assert call["on_error"]["task_name"] == "cb.error"+ assert "link" not in call and "link_error" not in callclass TestFairnessKeyComposesWithHeaders:@@ -154,7 +138,7 @@ class TestFairnessKeyComposesWithHeaders:class TestExecuteExtractionDispatchInventory:"""Canary: ``execute_extraction`` must only be dispatched via- ``ExecutionDispatcher``. Raw **string-literal**+ ``PgExecutionDispatcher``. Raw **string-literal**``*.send_task("execute_extraction", ...)`` elsewhere is forbidden.Known blind spots (deliberate — widening adds AST resolution cost@@ -174,9 +158,10 @@ class TestExecuteExtractionDispatchInventory:]assert offenders == [], ("Production code calls ``*.send_task(\"execute_extraction\", ...)`` "- "outside ``ExecutionDispatcher``. Use "- "``ExecutionDispatcher.dispatch(...)`` instead so fairness "- "headers and queue routing stay consistent. (Detection "+ "outside ``PgExecutionDispatcher``. Use "+ "``get_executor_dispatcher().dispatch(...)`` instead so queue routing "+ "and the reply-key contract stay consistent — and because a raw "+ "send_task publishes to a broker nothing drains. (Detection ""covers string-literal task names only; constant references, ""f-strings, and apply_async are blind spots.) Found:\n "+ "\n ".join(offenders)diff --git a/workers/tests/test_extraction_pipeline_contracts.py b/workers/tests/test_extraction_pipeline_contracts.pyindex 4eecb2b44..53e2fab08 100644--- a/workers/tests/test_extraction_pipeline_contracts.py+++ b/workers/tests/test_extraction_pipeline_contracts.py@@ -698,20 +698,22 @@ class TestSanityResponseContracts:class TestSanityDispatcher:- """Full-chain dispatcher tests with Celery eager mode."""+ """Full-chain dispatcher tests, running the executor task in-process."""@patch(_PATCH_FS)@patch(_PATCH_X2TEXT)def test_dispatcher_dispatch_full_chain(self, mock_x2text_cls, mock_get_fs, eager_app):- """ExecutionDispatcher dispatches through Celery and returns result.+ """The dispatcher drives the executor task and returns its result.- Celery's ``send_task`` doesn't reliably use eager mode, so we- patch it to route through ``task.apply()`` instead — this still- exercises the full Dispatcher → task → orchestrator chain.+ ``EagerExecutorTransport`` runs ``execute_extraction`` in-process rather+ than enqueueing it, so this still exercises the full+ dispatcher → task → orchestrator chain without a queue or a database."""- from unstract.sdk1.execution.dispatcher import ExecutionDispatcher+ from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher++ from .executor_dispatch_fakes import EagerExecutorTransportmock_x2text = MagicMock()mock_x2text.process.return_value = _mock_process_response("dispatched")@@ -719,29 +721,21 @@ class TestSanityDispatcher:mock_x2text_cls.return_value = mock_x2textmock_get_fs.return_value = MagicMock()- task = eager_app.tasks["execute_extraction"]-- def eager_send_task(name, args=None, **kwargs):- return task.apply(args=args)-- with patch.object(eager_app, "send_task", side_effect=eager_send_task):- dispatcher = ExecutionDispatcher(celery_app=eager_app)- ctx = _extract_ctx()- result = dispatcher.dispatch(ctx, timeout=10)+ transport = EagerExecutorTransport(eager_app.tasks["execute_extraction"])+ dispatcher = PgExecutionDispatcher(transport)+ result = dispatcher.dispatch(_extract_ctx(), timeout=10)assert isinstance(result, ExecutionResult)assert result.success is Trueassert result.data[IKeys.EXTRACTED_TEXT] == "dispatched"-- def test_dispatcher_no_app_raises(self):- """ExecutionDispatcher(celery_app=None).dispatch() → ValueError."""- from unstract.sdk1.execution.dispatcher import ExecutionDispatcher-- dispatcher = ExecutionDispatcher(celery_app=None)- ctx = _extract_ctx()-- with pytest.raises(ValueError, match="No Celery app"):- dispatcher.dispatch(ctx)+ assert transport.queue == "celery_executor_legacy"++ # ``test_dispatcher_no_app_raises`` is gone with the Celery dispatcher it+ # characterised: ``PgExecutionDispatcher`` is transport-injected and never+ # raises by contract — an enqueue failure becomes ``ExecutionResult.failure``.+ # That contract is covered in ``test_executor_rpc.py``, which drives a+ # transport whose ``enqueue`` raises; duplicating it here would assert the+ # shared dispatcher's behaviour a second time in a suite about the pipeline.class TestSanityCrossCutting:diff --git a/workers/tests/test_legacy_transport_shim.py b/workers/tests/test_legacy_transport_shim.pynew file mode 100644index 000000000..034ff6ac6--- /dev/null+++ b/workers/tests/test_legacy_transport_shim.py@@ -0,0 +1,42 @@+"""Gating tests for the UN-4078 rolling-deploy shim.++Nothing reads the ``transport`` key after UN-4078, so only a test stops it being+dropped before the follow-up release — and dropping it strands executions that a+pre-UN-4078 worker picks up during a rolling deploy (it defaults an absent field+to "celery" and publishes to a broker with no consumers).++These must run in the DEFAULT lane: the ``PgBarrier.enqueue`` tests that also+cover the descriptor need a real Postgres and are skipped wherever one is absent,+so they cannot be the only guard. Delete this file with the shim's writes.+"""++from unstract.core.data_models import LEGACY_TRANSPORT_KEY, LEGACY_TRANSPORT_VALUE++from queue_backend.pg_barrier import build_callback_descriptor+++class TestCallbackDescriptorShim:+ def _descriptor(self):+ return build_callback_descriptor(+ task_name="process_batch_callback",+ kwargs={"execution_id": "exec-1", "organization_id": "org-1"},+ queue="file_processing_callback",+ fairness_headers=None,+ )++ def test_descriptor_carries_the_transport_shim(self):+ # A pre-UN-4078 consumer reads callback_descriptor.get("transport") and+ # fires the callback through Celery when it is missing.+ assert self._descriptor()[LEGACY_TRANSPORT_KEY] == LEGACY_TRANSPORT_VALUE++ def test_shim_value_is_never_celery(self):+ # "celery" is the one value that must never be written: it selects the+ # dead transport explicitly rather than by omission.+ assert self._descriptor()[LEGACY_TRANSPORT_KEY] == "pg_queue"++ def test_descriptor_keeps_its_other_wire_keys(self):+ d = self._descriptor()+ assert d["task_name"] == "process_batch_callback"+ assert d["queue"] == "file_processing_callback"+ assert d["kwargs"]["execution_id"] == "exec-1"+ assert d["fairness_headers"] is Nonediff --git a/workers/tests/test_orchestration_idempotency.py b/workers/tests/test_orchestration_idempotency.pyindex 4768ad583..d31f685b3 100644--- a/workers/tests/test_orchestration_idempotency.py+++ b/workers/tests/test_orchestration_idempotency.py@@ -29,29 +29,23 @@ from queue_backend.pg_barrier import ()from queue_backend.pg_queue.connection import create_pg_connection-# --- Layer 1: transport-gated gate (no DB) ---+# --- Layer 1: the duplicate-orchestration gate (no DB) ---class TestOrchestrationGate:- """``_should_skip_duplicate_orchestration`` — Celery never claims; on PG the- first delivery proceeds and a duplicate skips.+ """``_should_skip_duplicate_orchestration`` — the first delivery proceeds and+ a duplicate skips. The gate used to be transport-conditional (a no-op on+ Celery, which had no redelivery); with PG the only transport it always runs."""- def test_celery_transport_never_claims(self):- # The whole guard is PG-only: on Celery it must not even consult the claim- # (proves the Celery path is behaviorally untouched).- with patch("general.tasks.try_claim_orchestration") as claim:- assert _should_skip_duplicate_orchestration("e1", "org1", "celery") is False- claim.assert_not_called()-def test_pg_first_delivery_proceeds(self):with patch("general.tasks.try_claim_orchestration", return_value=True) as claim:- assert _should_skip_duplicate_orchestration("e1", "org1", "pg_queue") is False+ assert _should_skip_duplicate_orchestration("e1", "org1") is Falseclaim.assert_called_once_with("e1", "org1") # org stamped for the reaperdef test_pg_duplicate_delivery_skips(self):with patch("general.tasks.try_claim_orchestration", return_value=False):- assert _should_skip_duplicate_orchestration("e1", "org1", "pg_queue") is True+ assert _should_skip_duplicate_orchestration("e1", "org1") is Truedef test_claim_error_propagates(self):# A DB error in the claim must PROPAGATE, not be swallowed — a swallow-and-@@ -62,7 +56,7 @@ class TestOrchestrationGate:side_effect=psycopg2.OperationalError("db down"),):with pytest.raises(psycopg2.OperationalError):- _should_skip_duplicate_orchestration("e1", "org1", "pg_queue")+ _should_skip_duplicate_orchestration("e1", "org1")def test_skip_on_retry_logs_error_with_errorid(self, caplog):# A skip when retries > 0 is a SUPPRESSED retry (a prior attempt claimed@@ -71,7 +65,7 @@ class TestOrchestrationGate:with patch("general.tasks.try_claim_orchestration", return_value=False):with caplog.at_level(logging.ERROR, logger="general.tasks"):assert (- _should_skip_duplicate_orchestration("e1", "org1", "pg_queue", 2)+ _should_skip_duplicate_orchestration("e1", "org1", 2)is True)assert "ORCH_CLAIM_SUPPRESSED_RETRY" in caplog.text@@ -105,7 +99,6 @@ class TestOrchestratorShortCircuit:workflow_id="wf",execution_id="e-dup",hash_values_of_files={},- transport="pg_queue",)assert out == {"status": "skipped_duplicate_orchestration",@@ -142,7 +135,6 @@ class TestOrchestratorShortCircuit:workflow_id="wf",execution_id="e-x",hash_values_of_files={},- transport="pg_queue",)assert released == [] # claim never acquired → nothing released@@ -168,7 +160,6 @@ class TestOrchestratorShortCircuit:workflow_id="wf",execution_id="e-y",hash_values_of_files={},- transport="pg_queue",)assert released == ["e-y"] # claimed then failed → released@@ -205,49 +196,16 @@ class TestOrchestratorShortCircuit:workflow_id="wf",execution_id="e-term",hash_values_of_files={},- transport="pg_queue",)assert out["status"] == "skipped_terminal_execution"assert released == [] # claim kept → tombstone re-establishedapi_client.update_workflow_execution_status.assert_not_called() # never re-armed- def test_terminal_execution_not_skipped_on_celery(self):- # Celery has no redelivery, so the terminal guard is is_pg-gated → a no-op.- # A terminal status on the Celery path must NOT short-circuit: we prove the- # task runs PAST the guard by making the next step (tool validation, called- # right after the guard) raise a sentinel. Patched via the task's own- # globals — the bare-``tasks`` module the worker bootstrap loads, not- # ``general.tasks`` (the sys.path quirk; a module-attr patch would miss it).- api_client = MagicMock()- api_client.get_workflow_execution.return_value = MagicMock(- success=True,- data={- "execution": {- "status": tasks.ExecutionStatus.COMPLETED.value,- "workflow_id": "wf",- }- },- )-- def _sentinel(**_kwargs):- raise RuntimeError("reached past the guard")-- with patch.dict(- self._task_globals(), {"validate_workflow_tool_instances": _sentinel}- ):- with patch.object(- tasks.WorkerExecutionContext,- "setup_execution_context",- return_value=(MagicMock(), api_client),- ):- with pytest.raises(RuntimeError, match="reached past the guard"):- tasks.async_execute_bin_general(- schema_name="org",- workflow_id="wf",- execution_id="e-term-celery",- hash_values_of_files={},- transport="celery",- )+ # ``test_terminal_execution_not_skipped_on_celery`` is gone with the+ # transport: the terminal guard used to be is_pg-gated so the Celery path ran+ # past it. There is no un-guarded path left — the guard above+ # (``test_terminal_execution_skips_and_keeps_claim``) is now the only+ # behaviour.# --- Layer 2: claim primitive against real Postgres ---diff --git a/workers/tests/test_pg_barrier.py b/workers/tests/test_pg_barrier.pyindex f55408a2f..84bf8093e 100644--- a/workers/tests/test_pg_barrier.py+++ b/workers/tests/test_pg_barrier.py@@ -27,7 +27,7 @@ from queue_backend.pg_barrier import (_barrier_pg_decrement,_fire_barrier_callback,barrier_pg_abort,- barrier_pg_decr_and_check,+ _barrier_pg_decrement,claim_batch,run_batch_with_barrier,try_claim_orchestration,@@ -55,11 +55,14 @@ _CALLBACK = {def _mock_header_task():- """A header-task Signature whose clone() records link/link_error/apply_async."""- cloned = MagicMock(name="cloned_signature")- task = MagicMock(name="header_signature")- task.clone.return_value = cloned- return task, cloned+ """A header-task Signature for the fan-out, PG-shaped.++ Kept as a two-tuple so the many ``_mock_header_task()[0]`` call sites read+ unchanged; the second element is now unused (it was the ``.clone()`` mock the+ Celery ``.link`` path recorded against, and that path is gone).+ """+ sig = _pg_header()+ return sig, None# --- Layer 1: protocol shape + TTL (no DB) ---@@ -749,21 +752,40 @@ def _last_progress_age_seconds(conn, execution_id):class TestPgBarrierEnqueue:- def test_upsert_creates_row_and_attaches_links(self, barrier_db):+ def test_upsert_creates_row_and_dispatches_headers(self, barrier_db):tasks = [_mock_header_task() for _ in range(3)]- handle = PgBarrier().enqueue(- [t for t, _ in tasks],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-A"},- callback_queue="general",- app_instance=None,- )+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:+ handle = PgBarrier().enqueue(+ [t for t, _ in tasks],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-A"},+ callback_queue="general",+ app_instance=None,+ )assert handle.id == "exec-A"assert _row(barrier_db, "exec-A") == (3, [])- for _, cloned in tasks:- cloned.link.assert_called_once()- cloned.link_error.assert_called_once()- cloned.apply_async.assert_called_once()+ # One PG dispatch per header, each carrying its barrier context. There is+ # no .link/.link_error to assert on any more — the consumer decrements+ # in-body from the injected _barrier_context.+ assert mock_dispatch.call_count == 3+ for i, call in enumerate(mock_dispatch.call_args_list):+ assert call.kwargs["kwargs"]["_barrier_context"]["batch_index"] == i++ def test_callback_descriptor_still_carries_legacy_transport(self, barrier_db):+ # Rolling-deploy shim (UN-4078): a pre-UN-4078 file-processing pod reads+ # callback_descriptor["transport"] and fires the callback via Celery when+ # it is absent. Must stay written until the release after UN-4078.+ task, _ = _mock_header_task()+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:+ PgBarrier().enqueue(+ [task],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-LT"},+ callback_queue="general",+ app_instance=None,+ )+ ctx = mock_dispatch.call_args.kwargs["kwargs"]["_barrier_context"]+ assert ctx["callback_descriptor"]["transport"] == "pg_queue"def test_enqueue_sets_expires_cap_and_fresh_progress(self, barrier_db, monkeypatch):# enqueue stamps expires_at = now()+ttl (the absolute cap) AND@@ -799,32 +821,35 @@ class TestPgBarrierEnqueue:monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: barrier_db)tasks = [_mock_header_task() for _ in range(3)]- handle = PgBarrier().enqueue(- [t for t, _ in tasks],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-HEAL"},- callback_queue="general",- app_instance=None,- )+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:+ handle = PgBarrier().enqueue(+ [t for t, _ in tasks],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-HEAL"},+ callback_queue="general",+ app_instance=None,+ )assert handle.id == "exec-HEAL"assert dead.closed is True # stale conn discarded by _cursorassert _row(barrier_db, "exec-HEAL") == (3, []) # row landed once, remaining=N- for _, cloned in tasks: # every header dispatched exactly once- cloned.apply_async.assert_called_once()+ assert mock_dispatch.call_count == 3 # every header dispatched exactly once- def test_fairness_header_stamped(self, barrier_db):- task, cloned = _mock_header_task()- PgBarrier().enqueue(- [task],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-F"},- callback_queue="general",- app_instance=None,- fairness=FairnessKey(org_id="o", workload_type=WorkloadType.API),- )- headers = cloned.set.call_args.kwargs["headers"]- assert FAIRNESS_HEADER_NAME in headers+ def test_fairness_forwarded_to_dispatch(self, barrier_db):+ # Fairness reaches the queue row as a FairnessKey on the dispatch, not as+ # an AMQP header on a cloned signature (that was the .link path).+ task, _ = _mock_header_task()+ fairness = FairnessKey(org_id="o", workload_type=WorkloadType.API)+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:+ PgBarrier().enqueue(+ [task],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-F"},+ callback_queue="general",+ app_instance=None,+ fairness=fairness,+ )+ assert mock_dispatch.call_args.kwargs["fairness"] is fairnessdef test_upsert_overwrites_stale_state(self, barrier_db):# A prior run left a row at remaining=1 with results; the new enqueue@@ -889,17 +914,20 @@ class TestPgBarrierEnqueue:def test_mid_loop_dispatch_failure_deletes_row(self, barrier_db):good, _ = _mock_header_task()- bad, bad_cloned = _mock_header_task()- bad_cloned.apply_async.side_effect = RuntimeError("broker down")+ bad, _ = _mock_header_task()barrier = PgBarrier()- with pytest.raises(RuntimeError):- barrier.enqueue(- [good, bad],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-D"},- callback_queue="general",- app_instance=None,- )+ with patch(+ "queue_backend.dispatch.dispatch",+ side_effect=[None, RuntimeError("queue write failed")],+ ):+ with pytest.raises(RuntimeError):+ barrier.enqueue(+ [good, bad],+ callback_task_name="cb",+ callback_kwargs={"execution_id": "exec-D"},+ callback_queue="general",+ app_instance=None,+ )assert _row(barrier_db, "exec-D") is None # cleaned up@@ -917,7 +945,7 @@ def _seed(conn, execution_id, remaining, *, results="[]"):class TestDecrAndCheck:def test_pending_decrements_only(self, barrier_db):_seed(barrier_db, "exec-P", 3)- out = barrier_pg_decr_and_check(+ out = _barrier_pg_decrement({"f": 1}, execution_id="exec-P", callback_descriptor=_CALLBACK)assert out["status"] == "pending"@@ -940,7 +968,7 @@ class TestDecrAndCheck:" now() - interval '1 hour')")assert _last_progress_age_seconds(barrier_db, "exec-SL") > 3000 # ~1h stale- barrier_pg_decr_and_check(+ _barrier_pg_decrement({"f": 1}, execution_id="exec-SL", callback_descriptor=_CALLBACK)assert _last_progress_age_seconds(barrier_db, "exec-SL") < 5 # refreshed@@ -1006,9 +1034,9 @@ class TestDecrAndCheck:lambda **_k: create_pg_connection(env_prefix="TEST_DB_"),)- with patch("celery.current_app.signature") as sig:- sig.return_value.apply_async.return_value = MagicMock(id="cb-ptb")- out = barrier_pg_decr_and_check(+ with patch("queue_backend.dispatch.dispatch") as sig:+ sig.return_value = MagicMock(id="cb-ptb")+ out = _barrier_pg_decrement({"f": "x"}, execution_id="exec-PTB", callback_descriptor=_CALLBACK)@@ -1019,35 +1047,54 @@ class TestDecrAndCheck:def test_complete_fires_callback_with_aggregated_results(self, barrier_db):_seed(barrier_db, "exec-C", 1, results='[{"f": "a"}]')- with patch("celery.current_app.signature") as sig:- sig.return_value.apply_async.return_value = MagicMock(id="cb-task-1")- out = barrier_pg_decr_and_check(+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:+ mock_dispatch.return_value = MagicMock(id="cb-task-1")+ out = _barrier_pg_decrement({"f": "b"}, execution_id="exec-C", callback_descriptor=_CALLBACK)assert out["status"] == "complete"# Callback got the full aggregated list as its first positional arg.- assert sig.call_args.kwargs["args"] == [[{"f": "a"}, {"f": "b"}]]+ assert mock_dispatch.call_args.kwargs["args"] == [[{"f": "a"}, {"f": "b"}]]assert _row(barrier_db, "exec-C") is None # row deleted after dispatchdef test_complete_path_passes_fairness_header(self, barrier_db):_seed(barrier_db, "exec-FH", 1)- descriptor = {**_CALLBACK, "fairness_headers": {FAIRNESS_HEADER_NAME: {"o": 1}}}- with patch("celery.current_app.signature") as sig:- sig.return_value.apply_async.return_value = MagicMock(id="cb")- barrier_pg_decr_and_check(+ # A full header payload: the PG path *parses* it back into a FairnessKey+ # (the Celery path only forwarded the dict opaquely), so a partial one+ # would fail on the missing workload_type rather than assert anything.+ descriptor = {+ **_CALLBACK,+ "fairness_headers": {+ FAIRNESS_HEADER_NAME: {+ "org_id": "o",+ "workload_type": "api",+ "pipeline_priority": 5,+ }+ },+ }+ with patch("queue_backend.dispatch.dispatch") as mock_dispatch:+ mock_dispatch.return_value = MagicMock(id="cb")+ _barrier_pg_decrement({"f": 1}, execution_id="exec-FH", callback_descriptor=descriptor)- assert sig.call_args.kwargs["headers"] == {FAIRNESS_HEADER_NAME: {"o": 1}}+ # Reconstructed into a FairnessKey and handed to the dispatch, so the+ # callback rides the same org/priority its header tasks did.+ forwarded = mock_dispatch.call_args.kwargs["fairness"]+ assert forwarded.org_id == "o"+ assert forwarded.workload_type is WorkloadType.API+ assert forwarded.pipeline_priority == 5def test_callback_dispatch_failure_preserves_row(self, barrier_db):# The central failure-masking invariant: dispatch happens BEFORE the row- # is deleted, so an apply_async failure leaves the row in place (reclaimed- # by expiry) — guards against a delete-before-dispatch reorder.+ # is deleted, so a callback-dispatch failure leaves the row in place+ # (reclaimed by expiry) — guards against a delete-before-dispatch reorder._seed(barrier_db, "exec-CF", 1)- with patch("celery.current_app.signature") as sig:- sig.return_value.apply_async.side_effect = RuntimeError("broker down")+ with patch(+ "queue_backend.dispatch.dispatch",+ side_effect=RuntimeError("queue write failed"),+ ):with pytest.raises(RuntimeError):- barrier_pg_decr_and_check(+ _barrier_pg_decrement({"f": 1}, execution_id="exec-CF", callback_descriptor=_CALLBACK)assert _row(barrier_db, "exec-CF") is not None # row survives for TTL reclaim@@ -1056,7 +1103,7 @@ class TestDecrAndCheck:# jsonb_build_array() must append a list-shaped result as ONE element# (plain `||` would concatenate it). Guards that choice._seed(barrier_db, "exec-L", 2)- barrier_pg_decr_and_check(+ _barrier_pg_decrement([1, 2], execution_id="exec-L", callback_descriptor=_CALLBACK)remaining, results = _row(barrier_db, "exec-L")@@ -1068,7 +1115,7 @@ class TestDecrAndCheck:# be torn down (fail fast) rather than hang to expiry._seed(barrier_db, "exec-NB", 1)with pytest.raises(psycopg2.DataError):- barrier_pg_decr_and_check(+ _barrier_pg_decrement({"f": "bad\x00value"},execution_id="exec-NB",callback_descriptor=_CALLBACK,@@ -1081,8 +1128,8 @@ class TestDecrAndCheck:# fires — even when it would otherwise have hit remaining == 0._seed(barrier_db, "exec-DA", 1)barrier_pg_abort(execution_id="exec-DA") # header failed → row deleted- with patch("celery.current_app.signature") as sig:- out = barrier_pg_decr_and_check(+ with patch("queue_backend.dispatch.dispatch") as sig:+ out = _barrier_pg_decrement({"f": 1}, execution_id="exec-DA", callback_descriptor=_CALLBACK)assert out["status"] == "abandoned"@@ -1090,8 +1137,8 @@ class TestDecrAndCheck:def test_negative_remaining_does_not_fire(self, barrier_db):_seed(barrier_db, "exec-N", 0) # decrement → -1- with patch("celery.current_app.signature") as sig:- out = barrier_pg_decr_and_check(+ with patch("queue_backend.dispatch.dispatch") as sig:+ out = _barrier_pg_decrement({"f": 1}, execution_id="exec-N", callback_descriptor=_CALLBACK)assert out["status"] == "abandoned"@@ -1099,8 +1146,8 @@ class TestDecrAndCheck:assert _row(barrier_db, "exec-N") is Nonedef test_missing_row_does_not_fire(self, barrier_db):- with patch("celery.current_app.signature") as sig:- out = barrier_pg_decr_and_check(+ with patch("queue_backend.dispatch.dispatch") as sig:+ out = _barrier_pg_decrement({"f": 1}, execution_id="nope", callback_descriptor=_CALLBACK)assert out["status"] == "abandoned"@@ -1110,14 +1157,10 @@ class TestDecrAndCheck:_seed(barrier_db, "exec-U", 1)bad_result = {object()}with pytest.raises(TypeError):- barrier_pg_decr_and_check(+ _barrier_pg_decrement(bad_result, execution_id="exec-U", callback_descriptor=_CALLBACK)- def test_registered_under_canonical_name(self):- assert barrier_pg_decr_and_check.name == "barrier_pg_decr_and_check"--class TestDecrementCoreExtraction:"""The decrement logic lives in a plain ``_barrier_pg_decrement`` core so the9e PR 2c PG-consumed path can call it in-body (a PG-consumed task fires no@@ -1149,7 +1192,7 @@ class TestDecrementCoreExtraction:# forwarding test below, this would catch a renamed/reordered core param# — the core is NOT mocked away._seed(barrier_db, "exec-wrap", 3)- out = barrier_pg_decr_and_check(+ out = _barrier_pg_decrement({"f": 1}, execution_id="exec-wrap", callback_descriptor=_CALLBACK)assert out["status"] == "pending"@@ -1157,21 +1200,9 @@ class TestDecrementCoreExtraction:assert remaining == 2assert results == [{"f": 1}]- def test_worker_task_forwards_kwargs_verbatim(self):- # Complements the real-row test above by pinning the keyword-forwarding- # contract explicitly: the wrapper passes result + execution_id +- # callback_descriptor straight through, returning the core's result.- sentinel = {"status": "pending", "remaining": 9}- with patch.object(- pg_barrier, "_barrier_pg_decrement", return_value=sentinel- ) as core:- out = barrier_pg_decr_and_check(- {"f": 1}, execution_id="exec-d", callback_descriptor=_CALLBACK- )- assert out is sentinel- core.assert_called_once_with(- {"f": 1}, execution_id="exec-d", callback_descriptor=_CALLBACK- )+ # ``test_worker_task_forwards_kwargs_verbatim`` is gone with the+ # ``barrier_pg_decr_and_check`` @worker_task wrapper it pinned: there is no+ # wrapper left to forward anything, only the core called directly in-body.def test_open_transaction_on_shared_conn_raises(self):# The in-body contract is enforced loudly: a caller that enters with an@@ -1210,8 +1241,9 @@ class TestAbort:assert barrier_pg_abort.name == "barrier_pg_abort"def test_max_retries_zero(self):- # A Celery retry would replay the decrement and corrupt the count.- assert barrier_pg_decr_and_check.max_retries == 0+ # A retry would replay the teardown. (The decrement's no-replay contract+ # is enforced by its in-body caller, not a task option — its @worker_task+ # wrapper went with the Celery .link path.)assert barrier_pg_abort.max_retries == 0@@ -1236,15 +1268,15 @@ class TestDecrementAtomicityThroughTask:conns.append(conn)pg_barrier._local.conn = conn # this thread's own connectiontry:- out = barrier_pg_decr_and_check(+ out = _barrier_pg_decrement({"t": label}, execution_id="exec-Z", callback_descriptor=_CALLBACK)statuses[label] = out["status"]finally:pg_barrier._local.conn = None- with patch("celery.current_app.signature") as sig:- sig.return_value.apply_async.return_value = MagicMock(id="cb")+ with patch("queue_backend.dispatch.dispatch") as sig:+ sig.return_value = MagicMock(id="cb")threads = [threading.Thread(target=run, args=(x,)) for x in ("a", "b")]for t in threads:t.start()@@ -1252,7 +1284,7 @@ class TestDecrementAtomicityThroughTask:t.join(timeout=10)assert all(not t.is_alive() for t in threads)assert sorted(statuses.values()) == ["complete", "pending"]- assert sig.return_value.apply_async.call_count == 1 # single fire+ assert sig.call_count == 1 # single firefor c in conns:c.close()@@ -1292,7 +1324,6 @@ class TestPgFireAndForgetMode:callback_kwargs={"execution_id": "exec-pg"},callback_queue="general",app_instance=None,- transport="pg_queue",)assert mock_dispatch.call_count == 2 # one per header, no .linkfor i, call in enumerate(mock_dispatch.call_args_list):@@ -1302,7 +1333,6 @@ class TestPgFireAndForgetMode:ctx = call.kwargs["kwargs"]["_barrier_context"]assert ctx["execution_id"] == "exec-pg"assert ctx["batch_index"] == i- assert ctx["callback_descriptor"]["transport"] == "pg_queue"# The pre-existing kwarg on h0 survives alongside the injected context.assert mock_dispatch.call_args_list[0].kwargs["kwargs"]["pre_existing"] == "keep"@@ -1321,7 +1351,6 @@ class TestPgFireAndForgetMode:callback_kwargs={"execution_id": "exec-reuse"},callback_queue="general",app_instance=None,- transport="pg_queue",)with barrier_db.cursor() as cur:cur.execute(@@ -1432,7 +1461,7 @@ class TestPgFireAndForgetMode:assert remaining == 2def test_fire_barrier_callback_pg_self_chains_via_dispatch(self):- descriptor = {**_CALLBACK, "transport": "pg_queue"}+ descriptor = dict(_CALLBACK)with patch("queue_backend.dispatch.dispatch") as mock_dispatch:mock_dispatch.return_value = MagicMock(id="pg-cb-1")cb_id = _fire_barrier_callback(descriptor, [{"r": 1}])@@ -1444,7 +1473,7 @@ class TestPgFireAndForgetMode:# The PG callback carries the _pg_transport marker so the aggregating# callback can gate its at-least-once duplicate guard on it. The shared# descriptor must NOT be mutated (a copy is dispatched).- descriptor = {**_CALLBACK, "transport": "pg_queue"}+ descriptor = dict(_CALLBACK)with patch("queue_backend.dispatch.dispatch") as mock_dispatch:mock_dispatch.return_value = MagicMock(id="pg-cb")_fire_barrier_callback(descriptor, [{"r": 1}])@@ -1452,21 +1481,11 @@ class TestPgFireAndForgetMode:assert dispatched.get(pg_barrier.PG_TRANSPORT_CALLBACK_KWARG) is Trueassert pg_barrier.PG_TRANSPORT_CALLBACK_KWARG not in descriptor["kwargs"]- def test_fire_barrier_callback_legacy_omits_transport_marker(self):- # The Celery .link path must NOT inject the marker → the callback's PG- # guard stays a no-op on Celery (no redelivery to guard).- with patch("celery.current_app.signature") as sig:- sig.return_value.apply_async.return_value = MagicMock(id="celery-cb")- _fire_barrier_callback(_CALLBACK, [{"r": 1}])- passed = sig.call_args.kwargs["kwargs"]- assert pg_barrier.PG_TRANSPORT_CALLBACK_KWARG not in passed-def test_fire_barrier_callback_pg_carries_fairness(self):# greptile: the PG callback must ride the producer's org/priority (parity# with the Celery path), reconstructed from the stored fairness_headers.descriptor = {**_CALLBACK,- "transport": "pg_queue","fairness_headers": {FAIRNESS_HEADER_NAME: {"org_id": "org-9",@@ -1485,20 +1504,12 @@ class TestPgFireAndForgetMode:def test_fire_barrier_callback_pg_without_fairness_passes_none(self):# No producer key → None (dispatch writes neutral defaults), not a crash.- descriptor = {**_CALLBACK, "transport": "pg_queue"} # fairness_headers None+ descriptor = dict(_CALLBACK) # fairness_headers Nonewith patch("queue_backend.dispatch.dispatch") as mock_dispatch:mock_dispatch.return_value = MagicMock(id="pg-cb")_fire_barrier_callback(descriptor, [{"r": 1}])assert mock_dispatch.call_args.kwargs["fairness"] is None- def test_fire_barrier_callback_legacy_uses_celery(self):- # No backend marker → the .link-mode Celery dispatch (unchanged).- with patch("celery.current_app.signature") as sig:- sig.return_value.apply_async.return_value = MagicMock(id="celery-cb")- cb_id = _fire_barrier_callback(_CALLBACK, [{"r": 1}])- assert cb_id == "celery-cb"- sig.assert_called_once()-def test_abort_clears_dedup_markers(self, barrier_db):with barrier_db.cursor() as cur:cur.execute("DELETE FROM pg_batch_dedup")@@ -1542,7 +1553,6 @@ class TestPgFireAndForgetMode:"kwargs": {"execution_id": "exec-last"},"queue": "api_file_processing_callback","fairness_headers": None,- "transport": "pg_queue",}ctx = {"execution_id": "exec-last",@@ -1621,7 +1631,6 @@ class TestPgFireAndForgetMode:callback_kwargs={"execution_id": "exec-midfail"},callback_queue="general",app_instance=None,- transport="pg_queue",)assert _row(barrier_db, "exec-midfail") is None # row deleted on failurewith barrier_db.cursor() as cur:diff --git a/workers/tests/test_pg_callback_duplicate_guard.py b/workers/tests/test_pg_callback_duplicate_guard.pyindex 656067959..d7d7e4d67 100644--- a/workers/tests/test_pg_callback_duplicate_guard.py+++ b/workers/tests/test_pg_callback_duplicate_guard.py@@ -41,8 +41,9 @@ class TestCallbackAlreadyRan:class TestCoreCallbackGuard:- """``_process_batch_callback_core`` short-circuits a PG duplicate before any- side effect (using the status the context already carries), and only on PG.+ """``_process_batch_callback_core`` short-circuits a duplicate before any side+ effect, using the status the context already carries. Unconditional since+ UN-4078 — the wire marker is still popped, but nothing branches on it."""def _context(self, status):@@ -88,16 +89,22 @@ class TestCoreCallbackGuard:with pytest.raises(AssertionError, match="side effects must not run"):self._run(is_pg=True, status=ExecutionStatus.ERROR.value)- def test_celery_never_skips(self):- # No marker → guard skipped entirely; the callback proceeds even on a- # COMPLETED status — proving zero Celery regression.- with pytest.raises(AssertionError, match="side effects must not run"):- self._run(is_pg=False, status=ExecutionStatus.COMPLETED.value)+ def test_guard_holds_without_the_marker(self):+ # UN-4078: the guard no longer keys on the wire marker — PG is the only+ # transport, so a COMPLETED redelivery is skipped whether or not the+ # producer stamped it (an old producer that omits it must not re-fire+ # webhooks / billing).+ out, _extract, determine = self._run(+ is_pg=False, status=ExecutionStatus.COMPLETED.value+ )+ assert out["duplicate_callback_skipped"] is True+ determine.assert_not_called()class TestApiCallbackGuard:- """``process_batch_callback_api`` short-circuits a PG duplicate using the- execution status it already fetches (no extra round-trip).+ """``process_batch_callback_api`` short-circuits a duplicate using the+ execution status it already fetches (no extra round-trip). Unconditional+ since UN-4078."""def _run(self, *, is_pg: bool, status: str):@@ -131,6 +138,8 @@ class TestApiCallbackGuard:with pytest.raises(AssertionError, match="side effects must not run"):self._run(is_pg=True, status=ExecutionStatus.EXECUTING.value)- def test_celery_never_skips(self):- with pytest.raises(AssertionError, match="side effects must not run"):- self._run(is_pg=False, status=ExecutionStatus.COMPLETED.value)+ def test_guard_holds_without_the_marker(self):+ # UN-4078: see the core-callback twin — the guard is unconditional now.+ out, determine = self._run(is_pg=False, status=ExecutionStatus.COMPLETED.value)+ assert out["status"] == "skipped_duplicate_callback"+ determine.assert_not_called()diff --git a/workers/tests/test_pg_duplicate_write_guard.py b/workers/tests/test_pg_duplicate_write_guard.pyindex a91c62553..619b0bae5 100644--- a/workers/tests/test_pg_duplicate_write_guard.py+++ b/workers/tests/test_pg_duplicate_write_guard.py@@ -1,12 +1,11 @@-"""PG-only duplicate-destination-write guard + active-file TTL cap.+"""Duplicate-destination-write guard + active-file TTL cap.-On the PG (at-least-once) transport a batch can be re-run after a crash /+The queue is at-least-once, so a batch can be re-run after a crash /reaper-recovery, and such a re-run bypasses discovery's FileHistory filter. Thedestination path therefore re-checks FileHistory by hash right before the writeand skips it when the file already completed in a prior run — closing the-duplicate-write window. The guard is a no-op on the Celery transport (that path-is unchanged), honours use_file_history (ETL "rewrite every run"), and is-fail-open on any lookup error (never blocks a real write).+duplicate-write window. The guard honours use_file_history (ETL "rewrite every+run") and is fail-open on any lookup error (never blocks a real write)."""from __future__ import annotations@@ -17,7 +16,6 @@ import pytestfrom unstract.core.data_models import (ExecutionStatus,FileHashData,- WorkflowTransport,)from shared.models.file_processing import FileProcessingContext@@ -27,8 +25,6 @@ from shared.workflow.execution.active_file_manager import ()from shared.workflow.execution.service import WorkerWorkflowExecutionService-PG = WorkflowTransport.PG_QUEUE.value-CELERY = WorkflowTransport.CELERY.valuedef _svc(api_client=None):@@ -52,33 +48,26 @@ def _history(found=True, status=ExecutionStatus.COMPLETED.value):return {"found": found, "file_history": {"status": status} if found else None}-def _call(svc, *, transport, use_file_history=True, is_api=False, file_hash=None):- return svc._pg_destination_already_written(+def _call(svc, *, use_file_history=True, is_api=False, file_hash=None):+ return svc._destination_already_written(file_hash=file_hash or _file_hash(),- transport=transport,use_file_history=use_file_history,workflow_id="wf-1",is_api=is_api,)-def test_celery_transport_never_looks_up_or_skips():- svc = _svc()- assert _call(svc, transport=CELERY) is False- svc.api_client.get_file_history_by_cache_key.assert_not_called()--def test_use_file_history_false_never_looks_up_or_skips():# ETL/TASK "rewrite every run" contract — never skip its write.svc = _svc()- assert _call(svc, transport=PG, use_file_history=False) is False+ assert _call(svc, use_file_history=False) is Falsesvc.api_client.get_file_history_by_cache_key.assert_not_called()def test_pg_skips_when_completed_history_exists():svc = _svc()svc.api_client.get_file_history_by_cache_key.return_value = _history()- assert _call(svc, transport=PG) is True+ assert _call(svc) is True@pytest.mark.parametrize(@@ -92,7 +81,7 @@ def test_pg_skips_when_completed_history_exists():def test_pg_proceeds_when_not_completed(history):svc = _svc()svc.api_client.get_file_history_by_cache_key.return_value = history- assert _call(svc, transport=PG) is False+ assert _call(svc) is Falsedef test_pg_fails_open_and_logs_on_lookup_error():@@ -101,40 +90,33 @@ def test_pg_fails_open_and_logs_on_lookup_error():with mock.patch("shared.workflow.execution.service.logger") as log:- assert _call(svc, transport=PG) is False # fail-open+ assert _call(svc) is False # fail-openlog.error.assert_called_once() # loud (Sentry-visible), not a silent warningdef test_pg_proceeds_when_no_cache_key():svc = _svc()- assert _call(svc, transport=PG, file_hash=_file_hash(h=None)) is False+ assert _call(svc, file_hash=_file_hash(h=None)) is Falsesvc.api_client.get_file_history_by_cache_key.assert_not_called()def test_api_execution_matches_on_hash_only():svc = _svc()svc.api_client.get_file_history_by_cache_key.return_value = _history()- _call(svc, transport=PG, is_api=True)+ _call(svc, is_api=True)_, kwargs = svc.api_client.get_file_history_by_cache_key.call_argsassert kwargs["file_path"] is None-def test_file_processing_context_carries_transport():- ctx = FileProcessingContext(- file_data=mock.Mock(),- file_hash=_file_hash(),- api_client=mock.Mock(),- workflow_execution={},- transport=PG,- )- assert ctx.transport == PG-+def test_handle_destination_processing_consults_guard_and_skips_on_hit():+ """Real-call-site wiring: `_handle_destination_processing` consults the guard+ and, on a hit, must NOT call `destination.handle_output`.-def test_handle_destination_processing_forwards_transport_and_skips_on_hit():- """Real-call-site wiring: `_handle_destination_processing` forwards the- context's transport (not a non-existent `file_data.transport`) to the guard,- and on a hit it must NOT call `destination.handle_output`. This is exactly- the regression the original dead `getattr(file_data,'transport')` introduced.+ This pins the skip, which is the behaviour that closes the duplicate-write+ window. It previously also pinned that the transport reached the guard from+ the context rather than a non-existent `file_data.transport` — that argument+ is gone with the transport itself (UN-4078), so the guard now applies+ unconditionally and there is no threading left to get wrong."""svc = _svc()file_hash = _file_hash()@@ -144,7 +126,6 @@ def test_handle_destination_processing_forwards_transport_and_skips_on_hit():file_hash=file_hash,api_client=svc.api_client,workflow_execution={},- transport=PG,)workflow = mock.Mock()workflow.destination_config.to_dict.return_value = {}@@ -156,7 +137,7 @@ def test_handle_destination_processing_forwards_transport_and_skips_on_hit():"shared.workflow.execution.service.WorkerDestinationConnector") as WDC,mock.patch.object(- svc, "_pg_destination_already_written", return_value=True+ svc, "_destination_already_written", return_value=True) as guard,):WDC.from_config.return_value = destination@@ -172,7 +153,6 @@ def test_handle_destination_processing_forwards_transport_and_skips_on_hit():is_api=False,)- assert guard.call_args.kwargs["transport"] == PG # threaded from the contextassert guard.call_args.kwargs["use_file_history"] is Truedestination.handle_output.assert_not_called() # skipped, no duplicate writeassert result.processed is Falsediff --git a/workers/tests/test_pg_finalization_fixes.py b/workers/tests/test_pg_finalization_fixes.pyindex b1daad5c2..04b96464e 100644--- a/workers/tests/test_pg_finalization_fixes.py+++ b/workers/tests/test_pg_finalization_fixes.py@@ -1,7 +1,9 @@"""Unit tests for the PG finalization-strand fixes.-- L2: ``_update_execution_status_unified`` re-raises on the PG path (so the failed- finalization vt-redelivers / poison-drops) but swallows on the Celery path.+- L2: ``_update_execution_status_unified`` re-raises a failed write by default (so+ the failed finalization vt-redelivers / poison-drops). Since UN-4078 that is+ unconditional — the Celery swallow is gone — and ``raise_on_failure=False`` is+ reserved for the one caller already handling a failure.- L3: ``_backoff_with_jitter`` spreads retries across [base/2, base] (equal jitter)so concurrent workers don't retry in synchronized waves. Unconditional sinceUN-4046 — it was gated on ``pg_queue_enabled`` to keep the Celery flow on the@@ -44,10 +46,10 @@ class TestBackoffJitter:assert len({_backoff_with_jitter(1.0, 3) for _ in range(64)}) > 1-class TestPGFinalizationReraise:+class TestFinalizationReraise:AGG = {"total_files": 1, "successful_files": 1, "failed_files": 0}- def _run(self, is_pg: bool, fail: bool):+ def _run(self, raise_on_failure: bool, fail: bool):from callback.tasks import _update_execution_status_unifiedapi = MagicMock()@@ -61,25 +63,26 @@ class TestPGFinalizationReraise:final_status="COMPLETED",aggregated_results=self.AGG,organization_id="org-1",- is_pg=is_pg,+ raise_on_failure=raise_on_failure,),)- def test_pg_reraises_on_write_failure(self):- # The whole point: a failed finalization on PG must propagate so the- # message vt-redelivers instead of stranding the execution.- _api, call = self._run(is_pg=True, fail=True)+ def test_reraises_on_write_failure(self):+ # The whole point: a failed finalization must propagate so the message+ # vt-redelivers instead of stranding the execution.+ _api, call = self._run(raise_on_failure=True, fail=True)with pytest.raises(RuntimeError):call()- def test_celery_swallows_on_write_failure(self):- # Celery behavior is unchanged — swallow and return the error dict.- _api, call = self._run(is_pg=False, fail=True)+ def test_opt_out_swallows_on_write_failure(self):+ # raise_on_failure=False: the ERROR-marking write inside an except block,+ # where a raise would mask the original failure.+ _api, call = self._run(raise_on_failure=False, fail=True)result = call()assert result["status"] == "failed"- def test_success_path_returns_completed_for_pg(self):- api, call = self._run(is_pg=True, fail=False)+ def test_success_path_returns_completed(self):+ api, call = self._run(raise_on_failure=True, fail=False)result = call()assert result["status"] == "completed"api.update_workflow_execution_status.assert_called_once()diff --git a/workers/tests/test_pg_orchestration_failure.py b/workers/tests/test_pg_orchestration_failure.pyindex 75adc03a1..5d77f74ce 100644--- a/workers/tests/test_pg_orchestration_failure.py+++ b/workers/tests/test_pg_orchestration_failure.py@@ -6,9 +6,9 @@ counter reconciliation, and the guarantee that **neither a UI-logging hiccup nora status-update failure may disturb the caller's original exception/flow** (itmust never raise, so the handler's own re-raise / error-response always runs).-The Celery path is byte-identical by construction: this recorder is only invoked-on the ``is_pg_transport`` branch, so a Celery failure never reaches it (and so-never sends file aggregates).+It used to be reached only on the ``is_pg_transport`` branch, with the Celery+path keeping a bare status update; with the transport gone (UN-4078) it is the+single failure-recording path for both workers."""from __future__ import annotationsdiff --git a/workers/tests/test_plugin_loader.py b/workers/tests/test_plugin_loader.pyindex 9e682a8fb..4ac720cd9 100644--- a/workers/tests/test_plugin_loader.py+++ b/workers/tests/test_plugin_loader.py@@ -7,7 +7,7 @@ Verifies:4. ExecutorPluginLoader.get() discovers entry-point-based plugins (mocked)5. ExecutorPluginLoader.discover_executors() loads cloud executors (mocked)6. text_processor.add_hex_line_numbers()-7. ExecutionDispatcher._get_queue() naming convention+7. Queue-per-executor naming convention (QUEUE_PREFIX)8. Protocol classes importable and runtime-checkable9. executors/__init__.py triggers discover_executors()"""@@ -17,7 +17,7 @@ from unittest.mock import MagicMock, patchimport pytestfrom executor.executors.plugins.loader import ExecutorPluginLoaderfrom executor.executors.plugins.text_processor import add_hex_line_numbers-from unstract.sdk1.execution.dispatcher import ExecutionDispatcher+from unstract.workflow_execution.executor_rpc import QUEUE_PREFIX@pytest.fixture(autouse=True)@@ -232,35 +232,31 @@ class TestTextProcessor:class TestQueuePerExecutor:- def test_get_queue_legacy(self):- assert ExecutionDispatcher._get_queue("legacy") == "celery_executor_legacy"+ """Queue-per-executor naming, now owned solely by the PG executor RPC.- def test_get_queue_table(self):- assert ExecutionDispatcher._get_queue("table") == "celery_executor_table"-- def test_get_queue_smart_table(self):- assert (- ExecutionDispatcher._get_queue("smart_table") == "celery_executor_smart_table"- )-- def test_get_queue_simple_prompt_studio(self):- assert (- ExecutionDispatcher._get_queue("simple_prompt_studio")- == "celery_executor_simple_prompt_studio"- )+ The names are ``celery_executor_*`` for historical reasons and are NOT dead+ Celery surface: ``worker-pg-executor`` subscribes to exactly these strings+ (chart ``workerPgExecutor.env.WORKER_PG_QUEUE_CONSUMER_QUEUE``). Renaming the+ prefix without changing the chart would silently strand every tool execution,+ so these assertions pin the literal wire names rather than deriving them.+ """- def test_get_queue_agentic(self):- assert ExecutionDispatcher._get_queue("agentic") == "celery_executor_agentic"+ @pytest.mark.parametrize(+ "executor_name",+ ["legacy", "table", "smart_table", "simple_prompt_studio", "agentic"],+ )+ def test_queue_for_known_executor(self, executor_name):+ assert f"{QUEUE_PREFIX}{executor_name}" == f"celery_executor_{executor_name}"- def test_get_queue_arbitrary_name(self):+ def test_queue_for_arbitrary_name(self):"""Any executor_name works — no whitelist."""- assert ExecutionDispatcher._get_queue("my_custom") == "celery_executor_my_custom"+ assert f"{QUEUE_PREFIX}my_custom" == "celery_executor_my_custom"- def test_queue_name_enum_matches_dispatcher(self):- """QueueName.EXECUTOR matches what dispatcher generates for 'legacy'."""+ def test_queue_name_enum_matches_prefix(self):+ """QueueName.EXECUTOR matches what the PG dispatcher builds for 'legacy'."""from shared.enums.worker_enums import QueueName- assert QueueName.EXECUTOR.value == ExecutionDispatcher._get_queue("legacy")+ assert QueueName.EXECUTOR.value == f"{QUEUE_PREFIX}legacy"# ── 6. Protocol classes importable ──────────────────────────────────diff --git a/workers/tests/test_plugin_migration_regression.py b/workers/tests/test_plugin_migration_regression.pyindex 29d4d81e6..b2a5046b5 100644--- a/workers/tests/test_plugin_migration_regression.py+++ b/workers/tests/test_plugin_migration_regression.py@@ -8,11 +8,13 @@ from unittest.mock import MagicMock, patchimport pytestfrom unstract.sdk1.execution.context import ExecutionContext, Operation-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.sdk1.execution.executor import BaseExecutorfrom unstract.sdk1.execution.orchestrator import ExecutionOrchestratorfrom unstract.sdk1.execution.registry import ExecutorRegistryfrom unstract.sdk1.execution.result import ExecutionResult+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher++from .executor_dispatch_fakes import FakeExecutorTransport, queue_for# ---------------------------------------------------------------------------# Fixtures@@ -436,15 +438,13 @@ class TestPluginDiscoveryToDispatchFlow:# Step 2: Verify registrationassert "discovered" in ExecutorRegistry.list_executors()- # Step 3: Dispatch via mock Celery- mock_app = MagicMock()- mock_result = MagicMock()- mock_result.get.return_value = ExecutionResult(- success=True, data={"output": "discovered_result"}- ).to_dict()- mock_app.send_task.return_value = mock_result-- dispatcher = ExecutionDispatcher(celery_app=mock_app)+ # Step 3: Dispatch via a fake executor transport+ transport = FakeExecutorTransport(+ result=ExecutionResult(+ success=True, data={"output": "discovered_result"}+ ).to_dict()+ )+ dispatcher = PgExecutionDispatcher(transport)ctx = ExecutionContext(executor_name="discovered",operation="custom_op",@@ -458,8 +458,7 @@ class TestPluginDiscoveryToDispatchFlow:assert result.data["output"] == "discovered_result"# Step 5: Verify queue routing- call_kwargs = mock_app.send_task.call_args- assert call_kwargs.kwargs["queue"] == "celery_executor_discovered"+ assert transport.queue == "celery_executor_discovered"# ---------------------------------------------------------------------------@@ -481,7 +480,7 @@ class TestQueueRoutingAllExecutors:list(EXECUTOR_QUEUE_MAP.items()),)def test_queue_name_for_executor(self, executor_name, expected_queue):- assert ExecutionDispatcher._get_queue(executor_name) == expected_queue+ assert queue_for(executor_name) == expected_queue# ---------------------------------------------------------------------------diff --git a/workers/tests/test_prompt_studio_dispatch.py b/workers/tests/test_prompt_studio_dispatch.pyindex 0f21605e9..9798215ca 100644--- a/workers/tests/test_prompt_studio_dispatch.py+++ b/workers/tests/test_prompt_studio_dispatch.py@@ -1,16 +1,16 @@"""IDE path integration tests through executor chain.-Phase 4 replaces PromptTool HTTP calls in PromptStudioHelper with-ExecutionDispatcher → executor worker → LegacyExecutor.+Phase 4 replaced PromptTool HTTP calls in PromptStudioHelper with a dispatch to+the executor worker → LegacyExecutor.-These tests build the EXACT payloads that prompt_studio_helper.py-now sends via ExecutionDispatcher, push them through the full Celery-eager-mode chain, and verify the results match what the IDE expects.+These tests build the EXACT payloads that prompt_studio_helper.py now sends,+push them through the full chain with the executor task running in-process, and+verify the results match what the IDE expects.This validates the full contract:prompt_studio_helper builds payload→ ExecutionContext(execution_source="ide", ...)- → Celery task → LegacyExecutor._handle_X()+ → execute_extraction task → LegacyExecutor._handle_X()→ ExecutionResult → result.data used by IDEAll tests use execution_source="ide" to match the real IDE path.@@ -23,9 +23,11 @@ from executor.executors.constants import (PromptServiceConstants as PSKeys,)from unstract.sdk1.execution.context import ExecutionContext-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.sdk1.execution.registry import ExecutorRegistryfrom unstract.sdk1.execution.result import ExecutionResult+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher++from .executor_dispatch_fakes import EagerExecutorTransport# ---------------------------------------------------------------------------# Patch targets (same as Phase 2 sanity)@@ -654,41 +656,28 @@ class TestIDESinglePass:class TestIDEDispatcherIntegration:- """Test ExecutionDispatcher dispatch() with IDE payloads in eager mode.+ """Dispatcher round trips with IDE payloads, running the task in-process.- Celery's send_task() doesn't work with eager mode for AsyncResult.get(),- so we patch send_task to delegate to task.apply() instead.+ ``EagerExecutorTransport`` runs ``execute_extraction`` synchronously instead+ of enqueueing it, so each test still covers dispatcher → task → executor →+ result. This replaces the old ``send_task``-patching dance, which existed+ only because Celery's eager mode does not support ``AsyncResult.get()``."""- @staticmethod- def _patch_send_task(eager_app):- """Patch send_task on eager_app to use task.apply()."""- original_send_task = eager_app.send_task-- def patched_send_task(name, args=None, kwargs=None, **opts):- task = eager_app.tasks[name]- return task.apply(args=args, kwargs=kwargs)-- eager_app.send_task = patched_send_task- return original_send_task-@patch(_PATCH_FS)@patch(_PATCH_X2TEXT)def test_dispatcher_extract_round_trip(self, mock_x2text_cls, mock_get_fs, eager_app):- """ExecutionDispatcher.dispatch() → extract → ExecutionResult."""+ """Dispatcher round trip → extract → ExecutionResult."""mock_x2text = MagicMock()mock_x2text.process.return_value = _mock_process_response("dispatcher extracted")mock_x2text.x2text_instance = MagicMock()mock_x2text_cls.return_value = mock_x2textmock_get_fs.return_value = MagicMock()- original = self._patch_send_task(eager_app)- try:- dispatcher = ExecutionDispatcher(celery_app=eager_app)- ctx = _ide_extract_ctx()- result = dispatcher.dispatch(ctx)- finally:- eager_app.send_task = original+ dispatcher = PgExecutionDispatcher(+ EagerExecutorTransport(eager_app.tasks["execute_extraction"])+ )+ result = dispatcher.dispatch(_ide_extract_ctx())assert result.success is Trueassert result.data["extracted_text"] == "dispatcher extracted"@@ -700,18 +689,15 @@ class TestIDEDispatcherIntegration:def test_dispatcher_answer_prompt_round_trip(self, mock_shim_cls, mock_deps, _mock_idx, _mock_plugin, eager_app):- """ExecutionDispatcher.dispatch() → answer_prompt → ExecutionResult."""+ """Dispatcher round trip → answer_prompt → ExecutionResult."""llm = _mock_llm("dispatcher answer")mock_deps.return_value = _mock_prompt_deps(llm)mock_shim_cls.return_value = MagicMock()- original = self._patch_send_task(eager_app)- try:- dispatcher = ExecutionDispatcher(celery_app=eager_app)- ctx = _ide_answer_prompt_ctx()- result = dispatcher.dispatch(ctx)- finally:- eager_app.send_task = original+ dispatcher = PgExecutionDispatcher(+ EagerExecutorTransport(eager_app.tasks["execute_extraction"])+ )+ result = dispatcher.dispatch(_ide_answer_prompt_ctx())assert result.success is Trueassert result.data["output"]["invoice_number"] == "dispatcher answer"@@ -724,18 +710,15 @@ class TestIDEDispatcherIntegration:def test_dispatcher_single_pass_round_trip(self, mock_shim_cls, mock_deps, _mock_idx, _mock_plugin, eager_app):- """ExecutionDispatcher.dispatch() → single_pass → ExecutionResult."""+ """Dispatcher round trip → single_pass → ExecutionResult."""llm = _mock_llm("sp dispatch")mock_deps.return_value = _mock_prompt_deps(llm)mock_shim_cls.return_value = MagicMock()- original = self._patch_send_task(eager_app)- try:- dispatcher = ExecutionDispatcher(celery_app=eager_app)- ctx = _ide_single_pass_ctx()- result = dispatcher.dispatch(ctx)- finally:- eager_app.send_task = original+ dispatcher = PgExecutionDispatcher(+ EagerExecutorTransport(eager_app.tasks["execute_extraction"])+ )+ result = dispatcher.dispatch(_ide_single_pass_ctx())assert result.success is Trueassert "revenue" in result.data["output"]@@ -743,7 +726,7 @@ class TestIDEDispatcherIntegration:@patch(_PATCH_FS)@patch(_PATCH_INDEX_DEPS)def test_dispatcher_index_round_trip(self, mock_deps, mock_get_fs, eager_app):- """ExecutionDispatcher.dispatch() → index → ExecutionResult."""+ """Dispatcher round trip → index → ExecutionResult."""mock_index_cls = MagicMock()mock_index = MagicMock()mock_index.generate_index_key.return_value = "doc-dispatch-idx"@@ -754,13 +737,10 @@ class TestIDEDispatcherIntegration:mock_deps.return_value = (mock_index_cls, MagicMock(), MagicMock())mock_get_fs.return_value = MagicMock()- original = self._patch_send_task(eager_app)- try:- dispatcher = ExecutionDispatcher(celery_app=eager_app)- ctx = _ide_index_ctx()- result = dispatcher.dispatch(ctx)- finally:- eager_app.send_task = original+ dispatcher = PgExecutionDispatcher(+ EagerExecutorTransport(eager_app.tasks["execute_extraction"])+ )+ result = dispatcher.dispatch(_ide_index_ctx())assert result.success is Trueassert result.data["doc_id"] == "doc-dispatch-idx"diff --git a/workers/tests/test_queue_backend_seam.py b/workers/tests/test_queue_backend_seam.pyindex 12e85f9fd..219e56009 100644--- a/workers/tests/test_queue_backend_seam.py+++ b/workers/tests/test_queue_backend_seam.py@@ -183,32 +183,20 @@ class TestPublicSurface:def test_all_exports(self):import queue_backend- # Phase 6a added Barrier / BarrierHandle / CeleryChordBarrier.- # Phase 6b adds RedisDecrBarrier + barrier_decr_and_check- # (registered as a Celery task on import) + the BarrierBackend- # enum + the get_barrier factory that the WORKER_BARRIER_BACKEND- # env flag drives.- # Phase 8a adds QueueBackend + select_backend — the queue-transport- # routing gate that the WORKER_PG_QUEUE_ENABLED_TASKS allow-list- # drives.- # The PG-queue barrier work adds the Postgres barrier surface:- # PgBarrier + its barrier_pg_decr_and_check / barrier_pg_abort tasks,- # selected when WORKER_BARRIER_BACKEND routes to the PG backend.+ # The surface after UN-4078 removed the Celery transport. What went:+ # CeleryChordBarrier / RedisDecrBarrier (the two other substrates),+ # BarrierBackend + get_barrier (the WORKER_BARRIER_BACKEND factory),+ # barrier_abort / barrier_decr_and_check (the Redis link tasks), and+ # barrier_pg_decr_and_check (the PG link-task wrapper — the decrement+ # now runs in-body, so only the abort is still a task).assert set(queue_backend.__all__) == {"Barrier",- "BarrierBackend","BarrierHandle",- "CeleryChordBarrier","FairnessKey","PgBarrier","QueueBackend",- "RedisDecrBarrier",- "barrier_abort",- "barrier_decr_and_check","barrier_pg_abort",- "barrier_pg_decr_and_check","dispatch",- "get_barrier","select_backend","worker_task",}diff --git a/workers/tests/test_redis_barrier.py b/workers/tests/test_redis_barrier.pydeleted file mode 100644index de098cf58..000000000--- a/workers/tests/test_redis_barrier.py+++ /dev/null@@ -1,972 +0,0 @@-"""Characterisation tests for ``RedisDecrBarrier`` (PG Queue Phase 6b).--The risky behaviour-change PR replacing Celery's chord aggregation-primitive with the labs-design ``DECR remaining`` + ``RPUSH results``-pattern (see ``queue_backend/redis_barrier.py``).--Three layers:--1. **Protocol shape** — ``RedisDecrBarrier`` satisfies the ``Barrier``- Protocol; the handle satisfies ``BarrierHandle``.-2. **Wire model** — ``enqueue`` initialises the per-execution counter- + results list with TTL, stamps fairness on each header task,- attaches the ``barrier_decr_and_check`` link, and dispatches via- ``apply_async``.-3. **Link aggregation** — the ``barrier_decr_and_check`` task runs the- atomic Lua script, fires the callback exactly when remaining reads- 0, passes the aggregated results, and respects fairness on the- callback signature.--Redis is mocked via ``MagicMock`` — the contract under test is the-sequence of Redis ops + their args, not Redis's own correctness.-A future integration test (against a real Redis) would belong in-``test_redis_barrier_integration.py``.-"""--from __future__ import annotations--import json-from unittest.mock import MagicMock, patch--import pytest--from queue_backend import (- Barrier,- BarrierHandle,- RedisDecrBarrier,- barrier_decr_and_check,-)-from queue_backend.fairness import FAIRNESS_HEADER_NAME, FairnessKey, WorkloadType-from queue_backend.redis_barrier import (- _KEY_TTL_DEFAULT_SECONDS,- _remaining_key,- _results_key,-)---# --- Protocol shape ------class TestRedisDecrBarrierProtocolShape:- def test_satisfies_barrier_protocol(self):- barrier: Barrier = RedisDecrBarrier()- assert callable(getattr(barrier, "enqueue", None))-- def test_handle_satisfies_barrier_handle(self):- from queue_backend.redis_barrier import _RedisBarrierHandle-- handle: BarrierHandle = _RedisBarrierHandle(id="exec-1")- assert handle.id == "exec-1"- assert isinstance(handle.id, str)---# --- Enqueue: wire model ------def redis_mock():- """Mock the Redis client used by ``RedisDecrBarrier``."""- return MagicMock(name="redis_client")---def barrier(redis_mock):- """``RedisDecrBarrier`` with an injected mock client.-- Production builds its client from env via ``create_redis_client``;- tests inject so no Redis is required.- """- return RedisDecrBarrier(redis_client=redis_mock)---class TestRedisDecrBarrierEnqueue:- def test_empty_header_returns_none_and_touches_no_redis(self, barrier, redis_mock):- """Mirrors the ``CeleryChordBarrier`` zero-header contract:- ``None`` is the *sole* signal of no-work-enqueued, and we- don't pay the Redis round-trip for it."""- result = barrier.enqueue(- [],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="q",- app_instance=MagicMock(),- )- assert result is None- # No SET / DELETE on the empty path — short-circuit before any- # Redis round-trip.- redis_mock.set.assert_not_called()- redis_mock.delete.assert_not_called()-- def test_missing_execution_id_raises(self, barrier):- """``RedisDecrBarrier`` uses ``execution_id`` as the key suffix- for ``remaining:{exec_id}`` / ``results:{exec_id}``. Without- it we can't isolate this execution's counter — fail loudly- instead of routing into a global namespace collision."""- header = [MagicMock(name="h1")]- with pytest.raises(ValueError, match=r"execution_id"):- barrier.enqueue(- header,- callback_task_name="cb",- # No execution_id.- callback_kwargs={"pipeline_id": "p-1"},- callback_queue="q",- app_instance=MagicMock(),- )-- def test_initialises_remaining_counter_with_ttl(self, barrier, redis_mock):- """``remaining:{exec_id}`` SET to ``len(header_tasks)`` with- TTL — belt-and-suspenders cleanup if a link never fires for- a task (e.g. worker crash mid-execution)."""- header = [MagicMock(name="h1"), MagicMock(name="h2"), MagicMock(name="h3")]- barrier.enqueue(- header,- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-42"},- callback_queue="q",- app_instance=MagicMock(),- )- redis_mock.set.assert_called_once_with(- _remaining_key("exec-42"),- 3,- ex=_KEY_TTL_DEFAULT_SECONDS,- )- # ``results:{exec_id}`` AND ``abort_lock:{exec_id}`` are DEL'd- # to clear any stale prior-run leftovers (e.g. from a retry- # reusing the same execution_id). Without DELing the abort- # lock here, a stale lock from a prior failed run would mask- # a retry's failure as success.- from queue_backend.redis_barrier import _abort_lock_key-- redis_mock.delete.assert_called_once_with(- _results_key("exec-42"),- _abort_lock_key("exec-42"),- )-- def test_link_attached_to_each_header_task(self, barrier, redis_mock):- """Every header task gets the ``barrier_decr_and_check`` link.- Cloning (not mutating the caller's list) preserves the same- guarantee ``CeleryChordBarrier`` makes for cross-tenant- signature reuse."""- h1 = MagicMock(name="h1")- h2 = MagicMock(name="h2")- barrier.enqueue(- [h1, h2],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="q",- app_instance=MagicMock(),- )- # Each task cloned, .link(...) called once on the clone,- # apply_async called once on the clone.- h1.clone.assert_called_once_with()- h2.clone.assert_called_once_with()- h1.clone.return_value.link.assert_called_once()- h2.clone.return_value.link.assert_called_once()- h1.clone.return_value.apply_async.assert_called_once_with()- h2.clone.return_value.apply_async.assert_called_once_with()- # The caller's signatures were NOT mutated (no .link on the- # originals).- h1.link.assert_not_called()- h2.link.assert_not_called()-- def test_fairness_header_stamped_on_each_header_task(self, barrier, redis_mock):- """``fairness.as_header()`` rides on every header signature as- an additive AMQP header — same wire shape as- ``CeleryChordBarrier``'s fairness plumbing."""- h1 = MagicMock(name="h1")- fairness = FairnessKey(org_id="org-x", workload_type=WorkloadType.API)- barrier.enqueue(- [h1],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="q",- app_instance=MagicMock(),- fairness=fairness,- )- # ``as_header()`` returns ``{FAIRNESS_HEADER_NAME: <dict>}``- # (a nested dict, not a JSON string — see fairness.py:62).- h1.clone.return_value.set.assert_called_once_with(- headers=fairness.as_header()- )-- def test_no_fairness_no_header_added(self, barrier, redis_mock):- """When ``fairness=None``, no ``headers=`` kwarg on the- signature — preserves wire equivalence for the- non-fairness call path."""- h1 = MagicMock(name="h1")- barrier.enqueue(- [h1],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="q",- app_instance=MagicMock(),- )- h1.clone.return_value.set.assert_not_called()-- def test_returns_handle_with_execution_id(self, barrier, redis_mock):- """``BarrierHandle.id`` is the execution id — what call sites- log for chord-id tracing. (Under ``CeleryChordBarrier`` this- was the chord aggregator's AsyncResult id; under- ``RedisDecrBarrier`` we don't have such a task, so we expose- the execution id which is a stable per-execution identifier- that the existing log consumers don't depend on the- underlying type of.)"""- result = barrier.enqueue(- [MagicMock()],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-42"},- callback_queue="q",- app_instance=MagicMock(),- )- assert result is not None- assert result.id == "exec-42"- assert isinstance(result.id, str)-- def test_setup_failure_raises(self, barrier, redis_mock):- """Any substrate failure (Redis down, network blip mid-setup)- propagates as an exception — matches the ``Barrier`` Protocol:- ``None`` is sole "no-op" signal, everything else raises."""- redis_mock.set.side_effect = ConnectionError("redis is down")- with pytest.raises(ConnectionError, match="redis is down"):- barrier.enqueue(- [MagicMock()],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="q",- app_instance=MagicMock(),- )-- def test_default_ttl_is_six_hours(self):- """Lock in the chosen default. Accidental shrinkage to a few- minutes would cause spurious callback fires on any execution- longer than the new value; accidental expansion to days would- leak orphaned keys. The default reflects the worst-case- ``FILE_PROCESSING_TASK_TIME_LIMIT`` (3h) x 2x margin."""- assert _KEY_TTL_DEFAULT_SECONDS == 6 * 60 * 60-- def test_ttl_overridable_via_env(self, barrier, redis_mock, monkeypatch):- """Operators with substantially longer (or shorter) workflows- can override the cleanup TTL via ``WORKER_BARRIER_KEY_TTL_SECONDS``.-- Set well above the default to confirm the env-driven path is- actually read (not the constant)."""- monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", str(3 * 24 * 3600))- barrier.enqueue(- [MagicMock()],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="q",- app_instance=MagicMock(),- )- # ``ex=`` on the counter SET should be the overridden 3-day value.- assert redis_mock.set.call_args.kwargs.get("ex") == 3 * 24 * 3600-- # Invalid-value handling tests live in ``TestTtlEnvValidation``- # below — they now assert ``raises`` (matching ``get_barrier()``'s- # loud-on-misconfig posture) rather than silent fallback.---# --- Link task: aggregation ------class TestBarrierDecrAndCheckLink:- """The ``barrier_decr_and_check`` worker task — the load-bearing- aggregator. Runs after each header task and either decrements +- aggregates (most calls) or fires the callback (the last call).- """-- @patch("queue_backend.redis_barrier._get_redis_client")- def test_pending_path_decrement_only(self, get_redis):- """When the post-decrement counter reads >0, no callback- dispatch — just ``RPUSH + DECR`` and return a pending status."""- redis_client = MagicMock(name="redis")- # Lua script returns [remaining, results-empty-list-since-pending].- # ``register_script`` returns a callable that returns the script result.- script = MagicMock(name="script", return_value=[2, []])- redis_client.register_script.return_value = script- get_redis.return_value = redis_client-- with patch("celery.current_app") as current_app:- # Invoke the underlying function directly (bypass- # Celery's @worker_task wrapper for unit testing).- result = barrier_decr_and_check.run(- {"batch_id": "b1", "status": "ok"},- execution_id="exec-1",- callback_descriptor={- "task_name": "cb",- "kwargs": {"execution_id": "exec-1"},- "queue": "q",- "fairness_headers": None,- },- )-- # Lua script invoked with the right keys + JSON-serialised- # result + TTL seconds for the EXPIRE inside the script.- script.assert_called_once()- call_kwargs = script.call_args.kwargs- assert call_kwargs["keys"] == [- _remaining_key("exec-1"),- _results_key("exec-1"),- ]- assert json.loads(call_kwargs["args"][0]) == {"batch_id": "b1", "status": "ok"}- assert call_kwargs["args"][1] == _KEY_TTL_DEFAULT_SECONDS-- # No callback dispatch — remaining > 0.- current_app.signature.assert_not_called()- assert result == {"status": "pending", "remaining": 2}-- @patch("queue_backend.redis_barrier._get_redis_client")- def test_complete_path_fires_callback_with_aggregated_results(self, get_redis):- """When the post-decrement counter reads 0, the link reads the- full results list (returned by the Lua script in the same- atomic step), constructs the callback signature with the- aggregated list as the first arg, and dispatches via- ``apply_async``."""- redis_client = MagicMock(name="redis")- # Lua returns [0, [serialised result_1, result_2, result_3]].- aggregated_raw = [- json.dumps({"batch_id": "b1", "status": "ok"}),- json.dumps({"batch_id": "b2", "status": "ok"}),- json.dumps({"batch_id": "b3", "status": "ok"}),- ]- script = MagicMock(name="script", return_value=[0, aggregated_raw])- redis_client.register_script.return_value = script- get_redis.return_value = redis_client-- with patch("celery.current_app") as current_app:- cb_sig = MagicMock(name="callback_signature")- cb_sig.apply_async.return_value = MagicMock(id="callback-task-id-xyz")- current_app.signature.return_value = cb_sig-- result = barrier_decr_and_check.run(- {"batch_id": "b3", "status": "ok"},- execution_id="exec-42",- callback_descriptor={- "task_name": "process_batch_callback_api",- "kwargs": {- "execution_id": "exec-42",- "pipeline_id": "pipe-7",- "organization_id": "org-x",- },- "queue": "api_file_processing_callback",- "fairness_headers": None,- },- )-- # Callback signature constructed with aggregated results as- # first arg, callback kwargs forwarded, queue pinned.- current_app.signature.assert_called_once_with(- "process_batch_callback_api",- args=[- [- {"batch_id": "b1", "status": "ok"},- {"batch_id": "b2", "status": "ok"},- {"batch_id": "b3", "status": "ok"},- ],- ],- kwargs={- "execution_id": "exec-42",- "pipeline_id": "pipe-7",- "organization_id": "org-x",- },- queue="api_file_processing_callback",- )- cb_sig.apply_async.assert_called_once_with()- # DELETE is now called from Python AFTER apply_async succeeds- # (the Lua script no longer DELs in the ``== 0`` branch). This- # is the load-bearing deferred-cleanup invariant: a dispatch- # failure leaves the keys + TTL in place so the execution- # isn't stranded.- redis_client.delete.assert_called_once_with(- _remaining_key("exec-42"),- _results_key("exec-42"),- )-- assert result["status"] == "complete"- assert result["callback_task_id"] == "callback-task-id-xyz"- assert result["aggregated_count"] == 3-- @patch("queue_backend.redis_barrier._get_redis_client")- def test_complete_path_passes_fairness_header_to_callback(self, get_redis):- """Fairness rides the callback signature too — same wire shape- as ``CeleryChordBarrier``'s callback fairness plumbing."""- redis_client = MagicMock(name="redis")- script = MagicMock(name="script", return_value=[0, [json.dumps({"x": 1})]])- redis_client.register_script.return_value = script- get_redis.return_value = redis_client-- # Match the wire shape produced by ``FairnessKey.as_header()`` —- # a nested dict, not a JSON string (see fairness.py:62).- fairness_headers = {- FAIRNESS_HEADER_NAME: {- "org_id": "org-x",- "workload_type": "non_api",- "pipeline_priority": 5,- }- }- with patch("celery.current_app") as current_app:- current_app.signature.return_value = MagicMock(- apply_async=MagicMock(return_value=MagicMock(id="cb-id"))- )-- barrier_decr_and_check.run(- {"x": 1},- execution_id="exec-1",- callback_descriptor={- "task_name": "cb",- "kwargs": {"execution_id": "exec-1"},- "queue": "q",- "fairness_headers": fairness_headers,- },- )-- # ``headers=fairness_headers`` MUST be on the signature kwargs.- assert current_app.signature.call_args.kwargs.get("headers") == fairness_headers-- @patch("queue_backend.redis_barrier._get_redis_client")- def test_unserialisable_result_raises_loudly(self, get_redis):- """A header task returning something json.dumps can't handle- is a typed-boundary regression. Raise loudly so the- execution surfaces an error rather than silently dropping the- result."""- redis_client = MagicMock(name="redis")- get_redis.return_value = redis_client-- # ``complex`` is a classic JSON-unserialisable leaf — no- # ``__json__`` / ``__str__``-with-meaning, and ``default=str``- # isn't applied (we dropped it). A regressed- # ``BatchExecutionResult.to_dict()`` producing such a leaf- # would land here. The test pins that the loud-failure- # ``try/except`` actually surfaces the regression rather than- # being a no-op.- with pytest.raises((TypeError, ValueError)):- barrier_decr_and_check.run(- {"bad": complex(1, 2)},- execution_id="exec-1",- callback_descriptor={- "task_name": "cb",- "kwargs": {"execution_id": "exec-1"},- "queue": "q",- "fairness_headers": None,- },- )-- def test_link_task_is_registered_with_celery_under_canonical_name(self):- """The link task must be importable + registered under- ``barrier_decr_and_check`` — any worker that processes the- link task's queue needs the name to resolve.-- Mixed-version rolling deploys depend on this: a worker- running 6b-old code without the link task registered would- fail with ``KeyError`` on the link's task name, which is a- loud failure (no silent drop)."""- assert barrier_decr_and_check.name == "barrier_decr_and_check"-- def test_link_max_retries_zero_prevents_double_decrement(self):- """The link task MUST have ``max_retries=0``. The Lua script- is atomic at the Redis level but the link itself is not- idempotent: a Celery retry would replay the script and- double-RPUSH + double-DECR, corrupting the aggregation- (counter hits 0 prematurely; results list has duplicates).- TTL-driven cleanup is the safety net for transient failures- rather than Celery autoretry."""- # Celery exposes ``max_retries`` on the task class.- assert barrier_decr_and_check.max_retries == 0-- @patch("queue_backend.redis_barrier._get_redis_client")- def test_negative_remaining_does_not_fire_callback(self, get_redis):- """The Lua script returns ``remaining < 0`` when the counter- was TTL-expired or already cleaned up. The Python side must- NOT fire the callback in that branch — otherwise every- subsequent task completion after a TTL expiry would dispatch- a spurious callback."""- redis_client = MagicMock(name="redis")- # Lua returns ``[-1, []]`` — the ``< 0`` branch where keys- # were already DEL'd (or never existed).- script = MagicMock(name="script", return_value=[-1, []])- redis_client.register_script.return_value = script- get_redis.return_value = redis_client-- with patch("celery.current_app") as current_app:- result = barrier_decr_and_check.run(- {"x": 1},- execution_id="exec-1",- callback_descriptor={- "task_name": "cb",- "kwargs": {"execution_id": "exec-1"},- "queue": "q",- "fairness_headers": None,- },- )-- # No callback dispatch — the abandoned branch fires.- current_app.signature.assert_not_called()- assert result["status"] == "abandoned"- assert result["remaining"] == -1-- @patch("queue_backend.redis_barrier._get_redis_client")- def test_callback_dispatch_failure_preserves_keys_for_ttl(self, get_redis):- """If ``apply_async`` raises after the Lua ``== 0`` branch- returned, the barrier keys MUST remain in Redis (with their- TTL still active) so the execution isn't stranded with no- recovery path.-- Without deferred DEL, the previous implementation DEL'd both- keys inside the Lua script before Python's ``apply_async``- ran. A dispatch failure (broker outage, serialisation error,- routing failure) then left the execution with: no keys, no- TTL, no Celery retry (``max_retries=0``), no link_error- fallback — strictly worse than the chord baseline.-- The fix: Lua's ``== 0`` branch returns the LRANGE results- WITHOUT DELing. Python DELs both keys only after- ``apply_async`` succeeds. A failure raises and the TTL- eventually cleans up the keys."""- redis_client = MagicMock(name="redis")- # ``== 0`` branch — script returns one result; Lua no longer DELs.- script = MagicMock(name="script", return_value=[0, [json.dumps({"x": 1})]])- redis_client.register_script.return_value = script- get_redis.return_value = redis_client-- with patch("celery.current_app") as current_app:- cb_sig = MagicMock(name="callback_signature")- # apply_async raises — broker outage, serialisation error, etc.- cb_sig.apply_async.side_effect = RuntimeError("broker is down")- current_app.signature.return_value = cb_sig-- with pytest.raises(RuntimeError, match=r"broker is down"):- barrier_decr_and_check.run(- {"x": 1},- execution_id="exec-1",- callback_descriptor={- "task_name": "cb",- "kwargs": {"execution_id": "exec-1"},- "queue": "q",- "fairness_headers": None,- },- )-- # DELETE MUST NOT have been called — the keys must remain in- # Redis (with their existing TTL) so:- # - Late-arriving link tasks (if any) cleanly hit the Lua- # ``< 0`` branch and exit via the abandoned path.- # - TTL eventually cleans the keys.- # - The failure is visible via Celery's standard task-failure- # channels rather than a silent infinite hang.- redis_client.delete.assert_not_called()---# --- TTL env validation ------class TestTtlEnvValidation:- """Misconfigured ``WORKER_BARRIER_KEY_TTL_SECONDS`` must raise.-- Matches the posture of ``get_barrier()`` on a typo'd- ``WORKER_BARRIER_BACKEND``: a misconfigured TTL shorter than- execution wall-clock is a correctness issue (spurious callback- fires per the Lua ``< 0`` branch), so operators get a loud- on-startup signal rather than silent degradation.- """-- def test_non_integer_raises(self, barrier, redis_mock, monkeypatch):- monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "not-an-int")- with pytest.raises(ValueError, match=r"not an integer"):- barrier.enqueue(- [MagicMock()],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="q",- app_instance=MagicMock(),- )-- def test_zero_raises(self, barrier, redis_mock, monkeypatch):- monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "0")- with pytest.raises(ValueError, match=r"positive integer"):- barrier.enqueue(- [MagicMock()],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="q",- app_instance=MagicMock(),- )-- def test_negative_raises(self, barrier, redis_mock, monkeypatch):- monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "-1")- with pytest.raises(ValueError, match=r"positive integer"):- barrier.enqueue(- [MagicMock()],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="q",- app_instance=MagicMock(),- )---# --- Mid-loop dispatch failure ------class TestDispatchFailureCleanup:- """If ``apply_async`` raises mid-fan-out, partially-dispatched- tasks' links would otherwise corrupt the counter and the orphan- state would leak. The barrier must DEL its keys before re-raising- so the in-flight links' DECR returns negative → Lua ``< 0``- branch cleanly abandons the execution.- """-- def test_mid_loop_dispatch_failure_cleans_up_keys(- self, barrier, redis_mock- ):- """Task K of N raises during ``apply_async`` — barrier must- DEL both keys before propagating the exception."""- h1 = MagicMock(name="h1")- h2 = MagicMock(name="h2")- h3_failing = MagicMock(name="h3_failing")- h3_failing.clone.return_value.apply_async.side_effect = (- RuntimeError("broker outage")- )-- with pytest.raises(RuntimeError, match=r"broker outage"):- barrier.enqueue(- [h1, h2, h3_failing],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="q",- app_instance=MagicMock(),- )-- # Initial SET + EXPIRE happened, then DEL on cleanup.- # The DEL call should reference BOTH barrier keys.- delete_calls = redis_mock.delete.call_args_list- # At least one of the DEL calls must include both barrier keys- # (the cleanup call), distinguishable from the initial- # ``delete(_results_key(...))`` stale-list clear.- cleanup_args = [- set(call.args) for call in delete_calls- if len(call.args) == 2- ]- assert any(- {_remaining_key("exec-1"), _results_key("exec-1")} == args- for args in cleanup_args- ), (- f"Expected DEL(remaining, results) in cleanup; saw "- f"delete calls: {[call.args for call in delete_calls]}"- )---# --- link_error: barrier_abort ------class TestBarrierAbortLinkError:- """``link_error`` propagates header-task failures.-- Without this, a failed header task would leave the counter stuck- above 0 and the execution would hang until TTL. ``barrier_abort``- DELs the barrier keys + logs the error; the outer task's error- handler is responsible for the workflow status update.- """-- def test_abort_registered_under_canonical_name(self):- """Mixed-version rolling deploys require the link_error task- to be importable by name on every worker that processes it."""- from queue_backend import barrier_abort-- assert barrier_abort.name == "barrier_abort"-- def test_link_error_attached_to_each_header_task(self, barrier, redis_mock):- """Every header task must get ``barrier_abort.s(...)`` as a- ``.link_error``. Without this, header failures don't- propagate and the execution hangs."""- h1 = MagicMock(name="h1")- h2 = MagicMock(name="h2")- barrier.enqueue(- [h1, h2],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-1"},- callback_queue="q",- app_instance=MagicMock(),- )- h1.clone.return_value.link_error.assert_called_once()- h2.clone.return_value.link_error.assert_called_once()-- @patch("queue_backend.redis_barrier._get_redis_client")- def test_abort_deletes_both_barrier_keys(self, get_redis):- """``barrier_abort`` cleans up the barrier state on header- failure — DELs both keys. The first call to ``barrier_abort``- for a given execution_id wins the SET NX race and proceeds- with cleanup + log."""- from queue_backend import barrier_abort-- redis_client = MagicMock(name="redis")- # ``set(...nx=True)`` returns True on first SET — this abort wins.- redis_client.set.return_value = True- get_redis.return_value = redis_client-- barrier_abort.run(- MagicMock(name="request"),- RuntimeError("header task crashed"),- "traceback string",- execution_id="exec-42",- )-- redis_client.delete.assert_called_once_with(- _remaining_key("exec-42"),- _results_key("exec-42"),- )-- @patch("queue_backend.redis_barrier._get_redis_client")- def test_concurrent_aborts_deduplicate(self, get_redis):- """N concurrent header-task failures attach the same- ``link_error`` → N ``barrier_abort`` executions for one- logical execution failure. Without dedup that would produce- N ``logger.error`` calls and N Sentry events. The SET NX- lock makes the first abort win; subsequent aborts early-exit- silently."""- from queue_backend import barrier_abort-- redis_client = MagicMock(name="redis")- # First call: SET NX returns True (first abort wins).- # Second call: SET NX returns False (lock already exists).- redis_client.set.side_effect = [True, False]- get_redis.return_value = redis_client-- result_1 = barrier_abort.run(- MagicMock(name="request"),- RuntimeError("header task 1 crashed"),- "traceback string",- execution_id="exec-42",- )- result_2 = barrier_abort.run(- MagicMock(name="request"),- RuntimeError("header task 2 crashed"),- "traceback string",- execution_id="exec-42",- )-- # First abort completed cleanup; second deduplicated.- assert result_1["status"] == "aborted"- assert result_2["status"] == "deduplicated"-- # DELETE called exactly ONCE — only the winning abort cleans- # up. The deduplicated path early-exits before DELETE.- redis_client.delete.assert_called_once_with(- _remaining_key("exec-42"),- _results_key("exec-42"),- )-- @patch("queue_backend.redis_barrier._get_redis_client")- def test_abort_uses_setnx_with_ttl(self, get_redis):- """The SET NX lock has an EX (TTL) — the lock key auto-- cleans alongside the barrier keys instead of leaking- forever on a failed execution."""- from queue_backend import barrier_abort- from queue_backend.redis_barrier import _KEY_TTL_DEFAULT_SECONDS-- redis_client = MagicMock(name="redis")- redis_client.set.return_value = True- get_redis.return_value = redis_client-- barrier_abort.run(- MagicMock(name="request"),- RuntimeError("crash"),- "traceback",- execution_id="exec-1",- )-- # SET called with nx=True + ex=<ttl>.- redis_client.set.assert_called_once()- call = redis_client.set.call_args- assert call.args[0] == "barrier:abort_lock:exec-1"- assert call.kwargs.get("nx") is True- assert call.kwargs.get("ex") == _KEY_TTL_DEFAULT_SECONDS-- @patch("queue_backend.redis_barrier._get_redis_client")- def test_abort_delete_failure_releases_lock(self, get_redis):- """If SET NX succeeds (this abort wins) but the subsequent- DELETE then raises (mid-abort Redis blip), the lock MUST be- released — otherwise sibling aborts hit the dedup early-exit- while the barrier keys are still alive, in-flight successful- tasks DECR to 0, and the callback fires with partial results- (silent-success-on-mid-abort-failure).-- Same correctness class as the stale-lock-on-retry bug,- reached via a different path: mid-abort failure instead of- cross-execution stale state.- """- from queue_backend import barrier_abort- from queue_backend.redis_barrier import _abort_lock_key-- redis_client = MagicMock(name="redis")- # SET NX returns True (this abort wins), DELETE then raises.- redis_client.set.return_value = True- # Two delete calls expected: the barrier-keys DEL that raises,- # then the lock-release DEL.- redis_client.delete.side_effect = [- ConnectionError("redis blip mid-abort"), # first DELETE raises- 1, # lock-release DEL succeeds (returns count of deleted keys)- ]- get_redis.return_value = redis_client-- with pytest.raises(ConnectionError, match=r"redis blip mid-abort"):- barrier_abort.run(- MagicMock(name="request"),- RuntimeError("header task crashed"),- "traceback",- execution_id="exec-1",- )-- # First DELETE was the barrier-keys cleanup; SECOND DELETE was- # the lock release (so siblings can re-acquire).- delete_calls = redis_client.delete.call_args_list- assert len(delete_calls) == 2, (- f"Expected 2 delete calls (barrier keys + lock release); "- f"got {len(delete_calls)}: {delete_calls}"- )- # The second call must target the abort_lock specifically.- assert delete_calls[1].args == (_abort_lock_key("exec-1"),), (- f"Expected lock-release DELETE; got args={delete_calls[1].args!r}"- )---# --- Retry with reused execution_id: stale abort_lock cleanup ------class TestAbortLockRetryCleanup:- """A retry that reuses ``execution_id`` must not be poisoned by a- stale ``abort_lock`` left behind by the previous failed run.-- Failure mode without the fix: execution A fails → ``barrier_abort``- writes ``abort_lock:foo`` (TTL 6h) → within 6h, execution B retries- with ``exec_id=foo`` → ``enqueue`` resets remaining/results but- abort_lock survives → if any header in B fails, its ``barrier_abort``- hits the stale lock and early-exits without DELing B's keys →- in-flight successful tasks DECR ``remaining`` to 0 → callback fires- with partial results → failed retry silently masked as successful.-- Fix: ``enqueue`` DELs the abort_lock alongside the results list at- setup time, so a retry starts with a clean slate.- """-- def test_enqueue_clears_stale_abort_lock(self, barrier, redis_mock):- """``enqueue`` must DEL the abort_lock key at setup so a- stale lock from a previous (failed) execution with the same- execution_id doesn't poison the retry's abort path."""- from queue_backend.redis_barrier import _abort_lock_key-- barrier.enqueue(- [MagicMock(name="h1")],- callback_task_name="cb",- callback_kwargs={"execution_id": "exec-foo"},- callback_queue="q",- app_instance=MagicMock(),- )- # The setup DEL must include the abort_lock key (alongside- # results) — otherwise a stale lock from a prior run with the- # same execution_id survives into this enqueue.- redis_mock.delete.assert_called_once_with(- _results_key("exec-foo"),- _abort_lock_key("exec-foo"),- )---# --- Process-local Redis client caching ------class TestRedisClientSingleton:- """``_get_redis_client()`` returns a process-cached client so the- high-frequency ``barrier_decr_and_check`` / ``barrier_abort``- invocations reuse the same ``ConnectionPool`` rather than spinning- up a fresh pool (and TCP handshakes) per task. At 1000-task- fan-out the difference is 1000 vs 1 pool initialisation.- """-- def test_get_redis_client_returns_same_instance(self):- """Repeated calls within one process must return the same- ``Redis`` instance — guarantees connection-pool reuse."""- from queue_backend import redis_barrier-- # Reset the cache so this test exercises the lazy-init path.- redis_barrier._redis_client_singleton = None- try:- with patch(- "queue_backend.redis_barrier.create_redis_client"- ) as factory:- factory.return_value = MagicMock(name="redis_client")- client_a = redis_barrier._get_redis_client()- client_b = redis_barrier._get_redis_client()- client_c = redis_barrier._get_redis_client()-- # All three calls return the same object.- assert client_a is client_b is client_c- # ``create_redis_client`` factory called exactly ONCE —- # subsequent calls hit the cache.- assert factory.call_count == 1- finally:- # Reset for downstream tests.- redis_barrier._redis_client_singleton = None-- def test_falls_back_to_canonical_redis_prefix_for_sentinel_ssl_inheritance(- self, monkeypatch- ):- """When no ``WORKER_BARRIER_REDIS_HOST`` is set, the barrier- MUST use the canonical ``REDIS_`` prefix directly so it- inherits the FULL config including ``REDIS_SENTINEL_MODE`` and- ``REDIS_SSL`` — not just HOST/PORT/PASSWORD/USER/DB.-- Without this fix, a Sentinel-backed deployment that relies on- the documented fallback would connect standalone (Sentinel- mode is per-prefix in ``create_redis_client`` and doesn't- cross-fall-back), failing every execution at flag-flip time."""- from queue_backend import redis_barrier-- # Ensure no barrier-specific HOST is set.- monkeypatch.delenv("WORKER_BARRIER_REDIS_HOST", raising=False)- redis_barrier._redis_client_singleton = None- try:- with patch(- "queue_backend.redis_barrier.create_redis_client"- ) as factory:- factory.return_value = MagicMock(name="redis_client")- redis_barrier._get_redis_client()-- # The factory MUST be called with env_prefix="REDIS_" so- # SENTINEL_MODE/SSL inherit from the canonical instance.- factory.assert_called_once()- assert factory.call_args.kwargs.get("env_prefix") == "REDIS_", (- f"Expected env_prefix='REDIS_' when no barrier-specific "- f"HOST is set (so Sentinel/SSL inherit); got "- f"env_prefix={factory.call_args.kwargs.get('env_prefix')!r}"- )- finally:- redis_barrier._redis_client_singleton = None-- def test_uses_barrier_prefix_when_dedicated_host_set(self, monkeypatch):- """When ``WORKER_BARRIER_REDIS_HOST`` IS set, the operator has- opted into a dedicated barrier Redis and is responsible for- the full barrier-prefix config (HOST + PORT + ... +- SENTINEL_MODE + SSL)."""- from queue_backend import redis_barrier-- monkeypatch.setenv("WORKER_BARRIER_REDIS_HOST", "dedicated-redis")- redis_barrier._redis_client_singleton = None- try:- with patch(- "queue_backend.redis_barrier.create_redis_client"- ) as factory:- factory.return_value = MagicMock(name="redis_client")- redis_barrier._get_redis_client()-- assert (- factory.call_args.kwargs.get("env_prefix")- == "WORKER_BARRIER_REDIS_"- ), (- f"Expected env_prefix='WORKER_BARRIER_REDIS_' when "- f"dedicated HOST is set; got "- f"env_prefix={factory.call_args.kwargs.get('env_prefix')!r}"- )- finally:- redis_barrier._redis_client_singleton = None---if __name__ == "__main__":- pytest.main([__file__, "-v"])diff --git a/workers/tests/test_simple_prompt_studio_operation.py b/workers/tests/test_simple_prompt_studio_operation.pyindex 926430e23..75dfef21e 100644--- a/workers/tests/test_simple_prompt_studio_operation.py+++ b/workers/tests/test_simple_prompt_studio_operation.py@@ -2,13 +2,14 @@executor registers, routes to its own queue, and LegacyExecutor rejects them."""-from unittest.mock import MagicMockfrom unstract.sdk1.execution.context import ExecutionContext-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.sdk1.execution.executor import BaseExecutorfrom unstract.sdk1.execution.registry import ExecutorRegistryfrom unstract.sdk1.execution.result import ExecutionResult+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher++from .executor_dispatch_fakes import FakeExecutorTransport, queue_for# ---------------------------------------------------------------------------# Mock SimplePromptStudioExecutor — registration and execution@@ -103,18 +104,16 @@ class TestSimplePromptStudioRegistration:class TestSPSQueueRouting:def test_sps_routes_to_correct_queue(self):- queue = ExecutionDispatcher._get_queue("simple_prompt_studio")+ queue = queue_for("simple_prompt_studio")assert queue == "celery_executor_simple_prompt_studio"def test_dispatch_sends_to_sps_queue(self):- mock_app = MagicMock()- mock_result = MagicMock()- mock_result.get.return_value = ExecutionResult(- success=True, data={"output": {"field": "value"}}- ).to_dict()- mock_app.send_task.return_value = mock_result-- dispatcher = ExecutionDispatcher(celery_app=mock_app)+ transport = FakeExecutorTransport(+ result=ExecutionResult(+ success=True, data={"output": {"field": "value"}}+ ).to_dict()+ )+ dispatcher = PgExecutionDispatcher(transport)ctx = ExecutionContext(executor_name="simple_prompt_studio",operation="sps_answer_prompt",@@ -124,19 +123,15 @@ class TestSPSQueueRouting:)dispatcher.dispatch(ctx)- mock_app.send_task.assert_called_once()- call_kwargs = mock_app.send_task.call_args- assert call_kwargs.kwargs.get("queue") == "celery_executor_simple_prompt_studio"+ assert transport.queue == "celery_executor_simple_prompt_studio"def test_dispatch_sps_index_to_correct_queue(self, tmp_path):- mock_app = MagicMock()- mock_result = MagicMock()- mock_result.get.return_value = ExecutionResult(- success=True, data={"output": "indexed"}- ).to_dict()- mock_app.send_task.return_value = mock_result-- dispatcher = ExecutionDispatcher(celery_app=mock_app)+ transport = FakeExecutorTransport(+ result=ExecutionResult(+ success=True, data={"output": "indexed"}+ ).to_dict()+ )+ dispatcher = PgExecutionDispatcher(transport)ctx = ExecutionContext(executor_name="simple_prompt_studio",operation="sps_index",@@ -146,9 +141,7 @@ class TestSPSQueueRouting:)dispatcher.dispatch(ctx)- mock_app.send_task.assert_called_once()- call_kwargs = mock_app.send_task.call_args- assert call_kwargs.kwargs.get("queue") == "celery_executor_simple_prompt_studio"+ assert transport.queue == "celery_executor_simple_prompt_studio"# ---------------------------------------------------------------------------diff --git a/workers/tests/test_smart_table_extract_operation.py b/workers/tests/test_smart_table_extract_operation.pyindex 608e5c8b7..971d2dbe6 100644--- a/workers/tests/test_smart_table_extract_operation.py+++ b/workers/tests/test_smart_table_extract_operation.py@@ -2,13 +2,14 @@its own queue, and LegacyExecutor rejects the operation."""-from unittest.mock import MagicMockfrom unstract.sdk1.execution.context import ExecutionContext-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.sdk1.execution.executor import BaseExecutorfrom unstract.sdk1.execution.registry import ExecutorRegistryfrom unstract.sdk1.execution.result import ExecutionResult+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher++from .executor_dispatch_fakes import FakeExecutorTransport, queue_for# ---------------------------------------------------------------------------# tasks.py log_component for smart_table_extract@@ -117,18 +118,16 @@ class TestSmartTableExtractorRegistration:class TestSmartTableQueueRouting:def test_smart_table_routes_to_correct_queue(self):- queue = ExecutionDispatcher._get_queue("smart_table")+ queue = queue_for("smart_table")assert queue == "celery_executor_smart_table"def test_dispatch_sends_to_smart_table_queue(self):- mock_app = MagicMock()- mock_result = MagicMock()- mock_result.get.return_value = ExecutionResult(- success=True, data={"output": "ok"}- ).to_dict()- mock_app.send_task.return_value = mock_result-- dispatcher = ExecutionDispatcher(celery_app=mock_app)+ transport = FakeExecutorTransport(+ result=ExecutionResult(+ success=True, data={"output": "ok"}+ ).to_dict()+ )+ dispatcher = PgExecutionDispatcher(transport)ctx = ExecutionContext(executor_name="smart_table",operation="smart_table_extract",@@ -138,9 +137,7 @@ class TestSmartTableQueueRouting:)dispatcher.dispatch(ctx)- mock_app.send_task.assert_called_once()- call_kwargs = mock_app.send_task.call_args- assert call_kwargs.kwargs.get("queue") == "celery_executor_smart_table"+ assert transport.queue == "celery_executor_smart_table"# ---------------------------------------------------------------------------diff --git a/workers/tests/test_summarize_operation.py b/workers/tests/test_summarize_operation.pyindex 4308387cc..9e8828e26 100644--- a/workers/tests/test_summarize_operation.py+++ b/workers/tests/test_summarize_operation.py@@ -3,18 +3,20 @@Verifies:1. Summarize operation exists and routes through LegacyExecutor2. Summarize executor_params contract matches _handle_summarize expectations-3. Dispatch routes summarize to celery_executor_legacy queue+3. Dispatch routes summarize to celery_executor_legacy queue (PG executor RPC)4. Summarize result has expected shape (data.data = summary text)-5. Full Celery chain for summarize operation+5. Full dispatch chain for summarize operation"""from unittest.mock import MagicMock, patchimport pytestfrom unstract.sdk1.execution.context import ExecutionContext, Operation-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.sdk1.execution.registry import ExecutorRegistryfrom unstract.sdk1.execution.result import ExecutionResult+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher++from .executor_dispatch_fakes import FakeExecutorTransport, queue_for# Patches_PATCH_GET_PROMPT_DEPS = (@@ -80,18 +82,16 @@ class TestSummarizeParamsContract:class TestSummarizeQueueRouting:def test_summarize_routes_to_legacy_queue(self):"""Summarize dispatches to celery_executor_legacy (LegacyExecutor)."""- queue = ExecutionDispatcher._get_queue("legacy")+ queue = queue_for("legacy")assert queue == "celery_executor_legacy"def test_dispatch_sends_summarize_to_legacy_queue(self):- mock_app = MagicMock()- mock_result = MagicMock()- mock_result.get.return_value = ExecutionResult(- success=True, data={"data": "Summary text here"}- ).to_dict()- mock_app.send_task.return_value = mock_result-- dispatcher = ExecutionDispatcher(celery_app=mock_app)+ transport = FakeExecutorTransport(+ result=ExecutionResult(+ success=True, data={"data": "Summary text here"}+ ).to_dict()+ )+ dispatcher = PgExecutionDispatcher(transport)ctx = ExecutionContext(executor_name="legacy",operation="summarize",@@ -108,9 +108,7 @@ class TestSummarizeQueueRouting:)result = dispatcher.dispatch(ctx)- mock_app.send_task.assert_called_once()- call_kwargs = mock_app.send_task.call_args- assert call_kwargs.kwargs.get("queue") == "celery_executor_legacy"+ assert transport.queue == "celery_executor_legacy"assert result.successassert result.data["data"] == "Summary text here"diff --git a/workers/tests/test_table_extract_operation.py b/workers/tests/test_table_extract_operation.pyindex 916921298..01d659f0e 100644--- a/workers/tests/test_table_extract_operation.py+++ b/workers/tests/test_table_extract_operation.py@@ -2,12 +2,13 @@routes to its own queue, and LegacyExecutor rejects the operation."""-from unittest.mock import MagicMockfrom unstract.sdk1.execution.context import ExecutionContext-from unstract.sdk1.execution.dispatcher import ExecutionDispatcherfrom unstract.sdk1.execution.registry import ExecutorRegistryfrom unstract.sdk1.execution.result import ExecutionResult+from unstract.workflow_execution.executor_rpc import PgExecutionDispatcher++from .executor_dispatch_fakes import FakeExecutorTransport, queue_for# ---------------------------------------------------------------------------# tasks.py log_component for table_extract@@ -122,19 +123,17 @@ class TestTableExtractorExecutorRegistration:class TestTableQueueRouting:def test_table_executor_routes_to_correct_queue(self):"""executor_name='table' routes to celery_executor_table queue."""- queue = ExecutionDispatcher._get_queue("table")+ queue = queue_for("table")assert queue == "celery_executor_table"def test_dispatch_sends_to_table_queue(self):- """ExecutionDispatcher sends table_extract to correct queue."""- mock_app = MagicMock()- mock_result = MagicMock()- mock_result.get.return_value = ExecutionResult(- success=True, data={"output": "ok"}- ).to_dict()- mock_app.send_task.return_value = mock_result-- dispatcher = ExecutionDispatcher(celery_app=mock_app)+ """The dispatcher sends table_extract to the correct queue."""+ transport = FakeExecutorTransport(+ result=ExecutionResult(+ success=True, data={"output": "ok"}+ ).to_dict()+ )+ dispatcher = PgExecutionDispatcher(transport)ctx = ExecutionContext(executor_name="table",operation="table_extract",@@ -144,9 +143,7 @@ class TestTableQueueRouting:)dispatcher.dispatch(ctx)- mock_app.send_task.assert_called_once()- call_kwargs = mock_app.send_task.call_args- assert call_kwargs.kwargs.get("queue") == "celery_executor_table"+ assert transport.queue == "celery_executor_table"# ---------------------------------------------------------------------------@@ -193,4 +190,4 @@ class TestEntryPointConfig:# This is a documentation/verification test — the entry point# in pyproject.toml maps 'table' to TableExtractorExecutor.# Verify the queue name matches.- assert ExecutionDispatcher._get_queue("table") == "celery_executor_table"+ assert queue_for("table") == "celery_executor_table"diff --git a/workers/tests/test_terminal_execution_guard.py b/workers/tests/test_terminal_execution_guard.pyindex bc301979b..dc334bbbe 100644--- a/workers/tests/test_terminal_execution_guard.py+++ b/workers/tests/test_terminal_execution_guard.py@@ -38,38 +38,36 @@ ACTIVE = [ExecutionStatus.PENDING.value, ExecutionStatus.EXECUTING.value]@pytest.mark.parametrize("status", TERMINAL)-def test_guard_raises_for_terminal_on_pg(status):+def test_guard_raises_for_terminal(status):with pytest.raises(_TerminalExecutionSkip) as exc:- _raise_if_execution_terminal({"status": status}, "exec-1", is_pg=True)+ _raise_if_execution_terminal({"status": status}, "exec-1")assert exc.value.execution_id == "exec-1"assert exc.value.status == status@pytest.mark.parametrize("status", ACTIVE)-def test_guard_passes_for_active_on_pg(status):+def test_guard_passes_for_active(status):# Must NOT raise for PENDING/EXECUTING.- _raise_if_execution_terminal({"status": status}, "exec-1", is_pg=True)+ _raise_if_execution_terminal({"status": status}, "exec-1")[email protected]("status", TERMINAL)-def test_guard_is_noop_on_celery_path(status):- # is_pg=False (Celery) → NEVER raises, even for terminal statuses. This is- # what keeps the Celery flow behaviorally unchanged.- _raise_if_execution_terminal({"status": status}, "exec-1", is_pg=False)+# ``test_guard_is_noop_on_celery_path`` is gone with the transport: the guard+# used to be is_pg-gated so the Celery flow never raised. It is unconditional+# now — the parametrised raise/pass tests above are the whole behaviour.-def test_missing_status_on_pg_warns_and_proceeds():+def test_missing_status_warns_and_proceeds():# A missing status is fail-open (no raise) but surfaced as a warning, since# it signals a degraded execution-fetch response rather than an active run.with mock.patch("file_processing.tasks.logger") as log:- _raise_if_execution_terminal({}, "exec-1", is_pg=True)+ _raise_if_execution_terminal({}, "exec-1")log.warning.assert_called_once()def test_guard_fires_at_real_call_site_before_status_write():- """Exercises the real wiring the mocked-setup tests skip: the is_pg forward- chain and the guard's placement in _setup_execution_context — after the- execution fetch, before the EXECUTING status write."""+ """Exercises the real wiring the mocked-setup tests skip: the guard's+ placement in _setup_execution_context — after the execution fetch, before+ the EXECUTING status write."""api_client = mock.Mock()api_client.get_workflow_execution.return_value = types.SimpleNamespace(success=True, data={"execution": {"status": ExecutionStatus.ERROR.value}}@@ -85,7 +83,7 @@ def test_guard_fires_at_real_call_site_before_status_write():mock.patch("file_processing.tasks.create_organization_context"),pytest.raises(_TerminalExecutionSkip),):- _setup_execution_context(batch_data, "task-1", is_pg=True)+ _setup_execution_context(batch_data, "task-1")# The guard fired BEFORE the EXECUTING status write.api_client.update_workflow_execution_status.assert_not_called()@@ -127,7 +125,7 @@ def test_run_batch_stages_skips_terminal_without_processing():"file_processing.tasks._process_individual_files") as process_files,):- result = _run_batch_stages({"any": "payload"}, "task-1", is_pg=True)+ result = _run_batch_stages({"any": "payload"}, "task-1")pre_create.assert_not_called()process_files.assert_not_called()@@ -197,7 +195,7 @@ def test_run_batch_stages_proceeds_when_not_terminal():return_value={"total_files": 1, "successful_files": 1, "failed_files": 0},),):- result = _run_batch_stages({"any": "payload"}, "task-1", is_pg=False)+ result = _run_batch_stages({"any": "payload"}, "task-1")pre_create.assert_called_once()process_files.assert_called_once()diff --git a/workers/tests/test_workflow_context_transport.py b/workers/tests/test_workflow_context_transport.pydeleted file mode 100644index b8db8d643..000000000--- a/workers/tests/test_workflow_context_transport.py+++ /dev/null@@ -1,51 +0,0 @@-"""Tests for the 9e transport field carried on ``WorkflowContextData``.--The transport a workflow execution rides is decided once at creation and-carried in the task payload; on the worker side it lands on the live-``WorkflowContextData`` so the fan-out (PR 2) can read it. PR 1 only adds the-field with a Celery default — these tests pin that it defaults and round-trips.-"""--from unittest.mock import MagicMock--from shared.models.execution_models import (- WorkerOrganizationContext,- WorkflowContextData,-)--from unstract.core.data_models import DEFAULT_WORKFLOW_TRANSPORT, WorkflowTransport---def _make_context(**overrides):- org_context = WorkerOrganizationContext(- organization_id="org-1",- api_client=MagicMock(),- )- kwargs = {- "workflow_id": "wf-1",- "workflow_name": "wf-name",- "workflow_type": "TASK",- "execution_id": "exec-1",- "organization_context": org_context,- "files": {},- }- kwargs.update(overrides)- return WorkflowContextData(**kwargs)---class TestWorkflowContextTransport:- def test_defaults_to_celery(self):- """A context built without a transport rides legacy Celery."""- assert _make_context().transport == DEFAULT_WORKFLOW_TRANSPORT- assert _make_context().transport == WorkflowTransport.CELERY.value-- def test_carries_pg_queue_when_set(self):- ctx = _make_context(transport=WorkflowTransport.PG_QUEUE.value)- assert ctx.transport == "pg_queue"-- def test_invalid_transport_coerced_to_celery(self):- """A garbage payload value (version skew / bad PG JSONB) must fail closed- at construction, not reach the PR 2 fan-out read."""- assert _make_context(transport="pg-queue").transport == "celery"- assert _make_context(transport="bogus").transport == "celery"- assert _make_context(transport=None).transport == "celery"