Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3452 [FEAT] Characterise the seams: dispatch + chord call sites (#1950)

ali60fb9cf34854
diff --git a/workers/tests/test_chord_sites_characterisation.py b/workers/tests/test_chord_sites_characterisation.py
new file mode 100644
index 000000000..7c9d653c2
--- /dev/null
+++ b/workers/tests/test_chord_sites_characterisation.py
@@ -0,0 +1,327 @@
+"""Characterisation tests for the two ``celery.chord`` call sites.
+
+These are the two places in ``workers/`` source that invoke ``chord(...)``
+directly. PR #13 will replace both with a transport-agnostic ``Barrier``
+abstraction matching the labs target architecture's ``DECR remaining:{exec_id}``
+pattern.
+
+This test suite locks down the **current** chord invocation contract so the
+migration can be proved equivalent. Chord is the highest-risk Celery construct
+called out in the PG Queue decision doc (silent task drops at ~130K-task scale)
+— characterising it before refactor is critical.
+
+Sites:
+1. ``shared/workflow/execution/orchestration_utils.py`` — ``WorkflowOrchestrationUtils.create_chord_execution`` (the centralised helper).
+2. ``api-deployment/tasks.py`` — inline chord inside ``_run_workflow_api``.
+ Site 2 is exercised only via the inventory test (full characterisation
+ requires heavy mocking of the enclosing 273-line function — out of scope
+ for a smoke/characterisation pass).
+"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+# --- Site 1: WorkflowOrchestrationUtils.create_chord_execution ---
+
+
+class TestCreateChordExecution:
+ """Characterise the centralised chord helper.
+
+ This is the easily-testable site. PR #13 should route both this helper
+ AND the inline call (Site 2) through the new Barrier abstraction.
+ """
+
+ def _make_app_instance(self):
+ """Build a Celery app-shaped mock with a working ``.signature(...)``."""
+ app = MagicMock()
+ # signature() returns an opaque "signature" object — represent it as a
+ # MagicMock that we can assert on by identity later.
+ app.signature.return_value = MagicMock(name="callback_signature")
+ return app
+
+ def test_empty_batch_tasks_returns_none_and_skips_chord(self):
+ """Zero files: helper short-circuits with None and never calls chord()."""
+ from shared.workflow.execution.orchestration_utils import (
+ WorkflowOrchestrationUtils,
+ )
+
+ app = self._make_app_instance()
+
+ with patch(
+ "shared.workflow.execution.orchestration_utils.chord"
+ ) as mock_chord:
+ result = WorkflowOrchestrationUtils.create_chord_execution(
+ batch_tasks=[],
+ callback_task_name="process_batch_callback",
+ callback_kwargs={"execution_id": "exec-1", "pipeline_id": "pipe-1"},
+ callback_queue="file_processing_callback",
+ app_instance=app,
+ )
+
+ assert result is None
+ mock_chord.assert_not_called()
+
+ def test_non_empty_batch_tasks_invokes_chord(self):
+ """Non-empty batch: chord(batch_tasks)(callback_signature) is called."""
+ from shared.workflow.execution.orchestration_utils import (
+ WorkflowOrchestrationUtils,
+ )
+
+ app = self._make_app_instance()
+ batch_tasks = [MagicMock(name="batch_task_1"), MagicMock(name="batch_task_2")]
+
+ with patch(
+ "shared.workflow.execution.orchestration_utils.chord"
+ ) as mock_chord:
+ # chord(batch_tasks) returns a chord object; calling it with the
+ # callback signature returns the chord result. Both calls must
+ # happen for this characterisation.
+ chord_obj = MagicMock(name="chord_object")
+ mock_chord.return_value = chord_obj
+
+ WorkflowOrchestrationUtils.create_chord_execution(
+ batch_tasks=batch_tasks,
+ callback_task_name="process_batch_callback",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="file_processing_callback",
+ app_instance=app,
+ )
+
+ # First call: chord(batch_tasks)
+ mock_chord.assert_called_once_with(batch_tasks)
+ # Second call: chord_obj(callback_signature) — applies the chord
+ chord_obj.assert_called_once_with(app.signature.return_value)
+
+ def test_callback_signature_is_built_with_correct_kwargs(self):
+ """The callback signature must be created with the exact task name,
+ kwargs, and queue passed in. PR #13 must preserve these."""
+ from shared.workflow.execution.orchestration_utils import (
+ WorkflowOrchestrationUtils,
+ )
+
+ app = self._make_app_instance()
+ batch_tasks = [MagicMock(name="batch")]
+ callback_kwargs = {
+ "execution_id": "exec-42",
+ "pipeline_id": "pipe-7",
+ "organization_id": "org-x",
+ }
+
+ with patch(
+ "shared.workflow.execution.orchestration_utils.chord"
+ ) as mock_chord:
+ mock_chord.return_value = MagicMock()
+ WorkflowOrchestrationUtils.create_chord_execution(
+ batch_tasks=batch_tasks,
+ callback_task_name="process_batch_callback_api",
+ callback_kwargs=callback_kwargs,
+ callback_queue="api_file_processing_callback",
+ app_instance=app,
+ )
+
+ app.signature.assert_called_once_with(
+ "process_batch_callback_api",
+ kwargs=callback_kwargs,
+ queue="api_file_processing_callback",
+ )
+
+ def test_returns_chord_result_object(self):
+ """The helper must return whatever ``chord(...)(...)`` returns —
+ callers depend on this return value to track the chord."""
+ from shared.workflow.execution.orchestration_utils import (
+ WorkflowOrchestrationUtils,
+ )
+
+ app = self._make_app_instance()
+
+ with patch(
+ "shared.workflow.execution.orchestration_utils.chord"
+ ) as mock_chord:
+ chord_obj = MagicMock()
+ chord_result = MagicMock(name="chord_result_object")
+ chord_obj.return_value = chord_result
+ mock_chord.return_value = chord_obj
+
+ result = WorkflowOrchestrationUtils.create_chord_execution(
+ batch_tasks=[MagicMock()],
+ callback_task_name="process_batch_callback",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="file_processing_callback",
+ app_instance=app,
+ )
+
+ assert result is chord_result
+
+ def test_chord_failure_is_re_raised_after_logging(self):
+ """If chord() raises, the helper logs and re-raises (not swallowed)."""
+ from shared.workflow.execution.orchestration_utils import (
+ WorkflowOrchestrationUtils,
+ )
+
+ app = self._make_app_instance()
+
+ with patch(
+ "shared.workflow.execution.orchestration_utils.chord"
+ ) as mock_chord:
+ mock_chord.side_effect = RuntimeError("broker exploded")
+
+ with pytest.raises(RuntimeError, match="broker exploded"):
+ WorkflowOrchestrationUtils.create_chord_execution(
+ batch_tasks=[MagicMock()],
+ callback_task_name="process_batch_callback",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="file_processing_callback",
+ app_instance=app,
+ )
+
+
+# --- Site 1 (mixin wrapper): WorkflowOrchestrationMixin.create_chord ---
+
+
+class TestWorkflowOrchestrationMixinCreateChord:
+ """Characterise the mixin wrapper around ``create_chord_execution``.
+
+ The mixin adds two unique behaviours over the static helper:
+ 1. Extracts ``self.app`` from the task context.
+ 2. Raises ``RuntimeError`` when no app is bound to the task.
+
+ Both must be preserved by PR #13 when it lifts the mixin to use Barrier.
+ """
+
+ def test_create_chord_extracts_app_from_self_and_delegates(self):
+ """The mixin must read ``self.app`` and forward it to the static helper."""
+ from shared.workflow.execution.orchestration_utils import (
+ WorkflowOrchestrationMixin,
+ WorkflowOrchestrationUtils,
+ )
+
+ # Build a synthetic task-like object carrying an `app` attribute.
+ task = type("FakeTask", (WorkflowOrchestrationMixin,), {})()
+ task.app = MagicMock(name="celery_app")
+ task.app.signature.return_value = MagicMock(name="callback_signature")
+
+ with patch.object(
+ WorkflowOrchestrationUtils, "create_chord_execution"
+ ) as mock_static:
+ mock_static.return_value = MagicMock(name="chord_result")
+ batch = [MagicMock()]
+ kwargs = {"execution_id": "exec-mixin"}
+ task.create_chord(
+ batch_tasks=batch,
+ callback_task_name="process_batch_callback",
+ callback_kwargs=kwargs,
+ callback_queue="file_processing_callback",
+ )
+
+ mock_static.assert_called_once_with(
+ batch,
+ "process_batch_callback",
+ kwargs,
+ "file_processing_callback",
+ task.app,
+ )
+
+ def test_create_chord_raises_when_no_app_bound(self):
+ """No ``self.app`` (e.g., called outside a Celery task context):
+ the mixin must raise ``RuntimeError`` rather than silently failing
+ or passing ``None`` downstream to chord()."""
+ from shared.workflow.execution.orchestration_utils import (
+ WorkflowOrchestrationMixin,
+ )
+
+ task = type("FakeTask", (WorkflowOrchestrationMixin,), {})()
+ # Deliberately do NOT set task.app — leave it absent.
+
+ with pytest.raises(RuntimeError, match="Celery app instance not available"):
+ task.create_chord(
+ batch_tasks=[MagicMock()],
+ callback_task_name="process_batch_callback",
+ callback_kwargs={"execution_id": "exec-1"},
+ callback_queue="file_processing_callback",
+ )
+
+
+# --- Cross-site invariant: chord call-site inventory ---
+
+
+class TestChordSiteInventory:
+ """Exactly two chord call sites must exist in workers/ source.
+
+ If a third appears, this test fails so PR #13's migration can't silently
+ miss it.
+ """
+
+ def test_only_two_known_chord_call_sites_in_workers(self):
+ """Count chord(...) invocations (not imports) in workers/ source."""
+ import pathlib
+ import re
+
+ workers_root = pathlib.Path(__file__).parent.parent
+ # Anchor to the top-level directory relative to workers_root so we
+ # don't accidentally exclude legitimately-named subdirectories like
+ # `workers/shared/tests_helpers/`.
+ skip_top_dirs = {"tests", "__pycache__", "htmlcov", ".venv"}
+
+ # Match `chord(` as a function call — excludes the bare import line
+ # `from celery import chord` and helper method names like `create_chord`.
+ # The regex requires `chord` to be preceded by start-of-line / whitespace
+ # / `=` / `(` (i.e., a true call expression), not as part of a longer
+ # identifier such as `create_chord`.
+ pattern = re.compile(r"(?:^|[\s=(])chord\(")
+
+ hits = []
+ for py in workers_root.rglob("*.py"):
+ rel_parts = py.relative_to(workers_root).parts
+ if rel_parts and rel_parts[0] in skip_top_dirs:
+ continue
+ text = py.read_text()
+ for line_no, line in enumerate(text.splitlines(), start=1):
+ if pattern.search(line):
+ hits.append(f"{py.relative_to(workers_root)}:{line_no}")
+
+ # Expected exactly two — in orchestration_utils.py and api-deployment/tasks.py.
+ assert len(hits) == 2, (
+ f"Expected exactly 2 chord(...) call sites in workers/, found "
+ f"{len(hits)}:\n " + "\n ".join(hits)
+ )
+ joined = " ".join(hits)
+ assert "shared/workflow/execution/orchestration_utils.py" in joined
+ assert "api-deployment/tasks.py" in joined
+
+ def test_chord_import_only_in_two_files(self):
+ """`from celery import chord` should appear in exactly the two files
+ that actually invoke chord — no other imports lurking."""
+ import pathlib
+ import re
+
+ workers_root = pathlib.Path(__file__).parent.parent
+ skip_top_dirs = {"tests", "__pycache__", "htmlcov", ".venv"}
+
+ pattern = re.compile(r"^\s*from\s+celery\s+import\s+chord\b")
+
+ hits = []
+ for py in workers_root.rglob("*.py"):
+ rel_parts = py.relative_to(workers_root).parts
+ if rel_parts and rel_parts[0] in skip_top_dirs:
+ continue
+ text = py.read_text()
+ for line_no, line in enumerate(text.splitlines(), start=1):
+ if pattern.search(line):
+ hits.append(f"{py.relative_to(workers_root)}:{line_no}")
+
+ assert len(hits) == 2, (
+ f"Expected `from celery import chord` in exactly 2 files, found "
+ f"{len(hits)}:\n " + "\n ".join(hits)
+ )
+ # Sanity: same files as the call-site canary above. If the imports
+ # ever migrate to different files while count stays at 2, this catches
+ # it — preventing a silent miss during the Barrier migration.
+ joined = " ".join(hits)
+ assert "shared/workflow/execution/orchestration_utils.py" in joined
+ assert "api-deployment/tasks.py" in joined
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/workers/tests/test_dispatch_sites_characterisation.py b/workers/tests/test_dispatch_sites_characterisation.py
new file mode 100644
index 000000000..95278a81b
--- /dev/null
+++ b/workers/tests/test_dispatch_sites_characterisation.py
@@ -0,0 +1,303 @@
+"""Characterisation tests for the two raw `current_app.send_task` dispatch sites.
+
+These are the only two places in workers/ that bypass `@shared_task`-based
+dispatch and call `current_app.send_task(...)` directly with a string task
+name. PR #8 (mass `@shared_task` -> `@worker_task` migration) will replace
+both with a unified `dispatch()` helper.
+
+This test suite locks down the **current** dispatch contract — task name,
+positional args, keyword args, and target queue — so the migration can
+be proved equivalent. It does NOT exercise the receiving tasks; it only
+captures what is dispatched.
+
+Sites characterised:
+1. ``shared/patterns/notification/helper.py:76`` — webhook notification
+2. ``scheduler/tasks.py:157`` — scheduled workflow async dispatch
+"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+
+# --- Site 1: shared/patterns/notification/helper.py ---
+
+
+class TestNotificationDispatchSite:
+ """Characterise ``send_notification_to_worker`` -> ``current_app.send_task``."""
+
+ def _make_payload(self):
+ """Build a minimal NotificationPayload-shaped mock.
+
+ The function under test only invokes ``payload.to_webhook_payload()``
+ and reads ``payload.pipeline_id`` (for log output). Anything else
+ about the payload object is irrelevant to dispatch behaviour.
+ """
+ payload = MagicMock()
+ payload.to_webhook_payload.return_value = {"event": "test_event", "id": 42}
+ payload.pipeline_id = "pipe-001"
+ return payload
+
+ def test_dispatch_task_name_and_queue(self):
+ from shared.patterns.notification.helper import send_notification_to_worker
+
+ with patch(
+ "shared.patterns.notification.helper.current_app"
+ ) as mock_app:
+ send_notification_to_worker(
+ url="https://example.com/hook",
+ payload=self._make_payload(),
+ auth_type="NONE",
+ auth_key=None,
+ auth_header=None,
+ )
+
+ mock_app.send_task.assert_called_once()
+ call = mock_app.send_task.call_args
+ assert call.args[0] == "send_webhook_notification"
+ assert call.kwargs["queue"] == "notifications"
+
+ def test_dispatch_positional_args_layout(self):
+ """Positional args MUST be [url, payload_dict, headers, timeout]."""
+ from shared.patterns.notification.helper import send_notification_to_worker
+
+ with patch(
+ "shared.patterns.notification.helper.current_app"
+ ) as mock_app:
+ send_notification_to_worker(
+ url="https://example.com/hook",
+ payload=self._make_payload(),
+ auth_type="BEARER",
+ auth_key="token-abc",
+ auth_header=None,
+ )
+
+ args = mock_app.send_task.call_args.kwargs["args"]
+ assert len(args) == 4
+ assert args[0] == "https://example.com/hook" # url
+ assert args[1] == {"event": "test_event", "id": 42} # payload_dict
+ assert args[2]["Authorization"] == "Bearer token-abc" # headers
+ assert args[3] == 10 # timeout (hard-coded)
+
+ def test_dispatch_kwargs_layout(self):
+ """Kwargs MUST contain max_retries, retry_delay, platform."""
+ from shared.patterns.notification.helper import send_notification_to_worker
+
+ with patch(
+ "shared.patterns.notification.helper.current_app"
+ ) as mock_app:
+ send_notification_to_worker(
+ url="https://example.com/hook",
+ payload=self._make_payload(),
+ auth_type="NONE",
+ auth_key=None,
+ auth_header=None,
+ max_retries=5,
+ platform="SLACK",
+ )
+
+ kwargs = mock_app.send_task.call_args.kwargs["kwargs"]
+ assert kwargs == {
+ "max_retries": 5,
+ "retry_delay": 10,
+ "platform": "SLACK",
+ }
+
+ def test_dispatch_returns_true_on_success(self):
+ from shared.patterns.notification.helper import send_notification_to_worker
+
+ with patch("shared.patterns.notification.helper.current_app"):
+ result = send_notification_to_worker(
+ url="https://example.com/hook",
+ payload=self._make_payload(),
+ auth_type="NONE",
+ auth_key=None,
+ auth_header=None,
+ )
+
+ assert result is True
+
+ def test_dispatch_returns_false_on_send_task_failure(self):
+ from shared.patterns.notification.helper import send_notification_to_worker
+
+ with patch(
+ "shared.patterns.notification.helper.current_app"
+ ) as mock_app:
+ mock_app.send_task.side_effect = RuntimeError("broker down")
+ result = send_notification_to_worker(
+ url="https://example.com/hook",
+ payload=self._make_payload(),
+ auth_type="NONE",
+ auth_key=None,
+ auth_header=None,
+ )
+
+ assert result is False
+
+
+# --- Site 2: scheduler/tasks.py ---
+
+
+class TestSchedulerDispatchSite:
+ """Characterise ``_execute_scheduled_workflow`` -> ``current_app.send_task``."""
+
+ def _make_context(self):
+ """Minimal ScheduledPipelineContext-shaped mock.
+
+ The dispatch site reads:
+ - context.organization_id, .workflow_id, .pipeline_id, .pipeline_name
+ - context.use_file_history
+ """
+ ctx = MagicMock()
+ ctx.organization_id = "org-test"
+ ctx.workflow_id = "wf-001"
+ ctx.pipeline_id = "pipe-007"
+ ctx.pipeline_name = "scheduled-test-pipeline"
+ ctx.use_file_history = False
+ return ctx
+
+ def _make_api_client(self, execution_id="exec-123"):
+ """Mock api_client whose create_workflow_execution returns a dict
+ with the expected execution_id."""
+ api = MagicMock()
+ api.create_workflow_execution.return_value = {"execution_id": execution_id}
+ return api
+
+ def test_dispatch_task_name(self):
+ from scheduler.tasks import _execute_scheduled_workflow
+
+ with patch("celery.current_app") as mock_app:
+ _execute_scheduled_workflow(self._make_api_client(), self._make_context())
+
+ mock_app.send_task.assert_called_once()
+ assert mock_app.send_task.call_args.args[0] == "async_execute_bin"
+
+ def test_dispatch_routes_to_general_queue(self):
+ from scheduler.tasks import _execute_scheduled_workflow
+ from shared.enums.worker_enums import QueueName
+
+ with patch("celery.current_app") as mock_app:
+ _execute_scheduled_workflow(self._make_api_client(), self._make_context())
+
+ assert mock_app.send_task.call_args.kwargs["queue"] == QueueName.GENERAL
+
+ def test_dispatch_positional_args_layout(self):
+ """Positional args MUST be [org_id, workflow_id, execution_id, {}, True]."""
+ from scheduler.tasks import _execute_scheduled_workflow
+
+ with patch("celery.current_app") as mock_app:
+ _execute_scheduled_workflow(
+ self._make_api_client(execution_id="exec-xyz"),
+ self._make_context(),
+ )
+
+ args = mock_app.send_task.call_args.kwargs["args"]
+ assert len(args) == 5
+ assert args[0] == "org-test" # organization_id (schema_name)
+ assert args[1] == "wf-001" # workflow_id
+ assert args[2] == "exec-xyz" # execution_id (from api_client return)
+ assert args[3] == {} # hash_values_of_files (always empty for scheduled)
+ assert args[4] is True # scheduled flag (always True here)
+
+ def test_dispatch_kwargs_layout(self):
+ """Kwargs MUST contain use_file_history and pipeline_id."""
+ from scheduler.tasks import _execute_scheduled_workflow
+
+ ctx = self._make_context()
+ ctx.use_file_history = True
+
+ with patch("celery.current_app") as mock_app:
+ _execute_scheduled_workflow(self._make_api_client(), ctx)
+
+ kwargs = mock_app.send_task.call_args.kwargs["kwargs"]
+ assert kwargs == {
+ "use_file_history": True,
+ "pipeline_id": "pipe-007",
+ }
+
+ def test_no_dispatch_when_execution_creation_fails(self):
+ """If api_client.create_workflow_execution returns no execution_id,
+ the function bails out and never calls send_task."""
+ from scheduler.tasks import _execute_scheduled_workflow
+ from shared.models.scheduler_models import SchedulerExecutionStatus
+
+ api = MagicMock()
+ api.create_workflow_execution.return_value = {} # no execution_id
+
+ with patch("celery.current_app") as mock_app:
+ result = _execute_scheduled_workflow(api, self._make_context())
+
+ # The dispatch contract: nothing is sent when execution creation fails.
+ mock_app.send_task.assert_not_called()
+ # And the function returns an error result (not raised exception).
+ assert result.status == SchedulerExecutionStatus.ERROR
+
+ def test_dispatch_returns_error_result_when_send_task_raises(self):
+ """If current_app.send_task raises (broker down, queue gone, etc.),
+ _execute_scheduled_workflow MUST catch it, log, and return a
+ SchedulerExecutionResult.error(...) — NOT propagate the exception.
+
+ PR #8 (dispatch migration) must preserve this semantic. If the new
+ dispatch() helper either swallows the error silently or re-raises a
+ different exception type, this test fails.
+ """
+ from scheduler.tasks import _execute_scheduled_workflow
+ from shared.models.scheduler_models import SchedulerExecutionStatus
+
+ with patch("celery.current_app") as mock_app:
+ mock_app.send_task.side_effect = RuntimeError("broker down")
+ result = _execute_scheduled_workflow(
+ self._make_api_client(), self._make_context()
+ )
+
+ # Exception was caught (not propagated).
+ assert result.status == SchedulerExecutionStatus.ERROR
+ # Execution itself was created before the dispatch failed, so the
+ # error result is about the dispatch, not the creation.
+ assert result.execution_id == "exec-123" or result.execution_id is None
+
+
+# --- Cross-site invariant ---
+
+
+class TestDispatchSiteInventory:
+ """If a third raw current_app.send_task site appears, this test breaks
+ so PR #8's migration can't silently miss it."""
+
+ def test_only_two_known_dispatch_sites_in_workers(self):
+ """Verify the count of raw current_app.send_task references in
+ workers/ source matches the two we have characterised."""
+ import pathlib
+ import re
+
+ workers_root = pathlib.Path(__file__).parent.parent
+ # Anchor skip to the top-level directory relative to workers_root so
+ # we don't accidentally exclude legitimately-named subdirectories
+ # (e.g. workers/shared/tests_helpers/).
+ skip_top_dirs = {"tests", "__pycache__", "htmlcov", ".venv"}
+
+ pattern = re.compile(r"current_app\.send_task\b")
+ hits = []
+ for py in workers_root.rglob("*.py"):
+ rel_parts = py.relative_to(workers_root).parts
+ if rel_parts and rel_parts[0] in skip_top_dirs:
+ continue
+ text = py.read_text()
+ for line_no, line in enumerate(text.splitlines(), start=1):
+ if pattern.search(line):
+ hits.append(f"{py.relative_to(workers_root)}:{line_no}")
+
+ # Expected exactly two — in helper.py and scheduler/tasks.py.
+ # If a third appears, this test fails so PR #8 doesn't miss it.
+ assert len(hits) == 2, (
+ f"Expected exactly 2 raw current_app.send_task sites in workers/, found "
+ f"{len(hits)}:\n " + "\n ".join(hits)
+ )
+ # Sanity: the two we know about
+ joined = " ".join(hits)
+ assert "shared/patterns/notification/helper.py" in joined
+ assert "scheduler/tasks.py" in joined
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])