LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3445 [GATED-FEAT] Complete the PostgreSQL queue transport: scheduler, metrics, log streaming and strand recovery (#2254)
alie452f778c7bb
diff --git a/backend/backend/internal_base_urls.py b/backend/backend/internal_base_urls.pyindex 0354a691a..30d4c8df2 100644--- a/backend/backend/internal_base_urls.py+++ b/backend/backend/internal_base_urls.py@@ -191,6 +191,13 @@ urlpatterns = [include("workflow_manager.workflow_execution_internal_urls"),name="workflow_execution_internal",),+ # Dashboard-metrics periodics (UN-3796) — called by the thin worker tasks the PG+ # scheduler fires, replacing Beat + workerMetrics.+ path(+ "v1/dashboard-metrics/",+ include("dashboard_metrics.internal_urls"),+ name="dashboard_metrics_internal",+ ),# Workflow management and pipeline APIspath("v1/workflow-manager/",diff --git a/backend/dashboard_metrics/internal_urls.py b/backend/dashboard_metrics/internal_urls.pynew file mode 100644index 000000000..bc9d698f7--- /dev/null+++ b/backend/dashboard_metrics/internal_urls.py@@ -0,0 +1,29 @@+"""Internal API URLs for the dashboard-metrics periodics (UN-3796).++Called by the thin worker tasks that the PG scheduler fires, replacing Beat ++``workerMetrics``. Mirrors the shape of ``execution_log_internal_urls``.+"""++from django.urls import path++from . import internal_views++app_name = "dashboard_metrics_internal"++urlpatterns = [+ path(+ "aggregate/",+ internal_views.AggregateMetricsAPIView.as_view(),+ name="aggregate_metrics",+ ),+ path(+ "cleanup/hourly/",+ internal_views.CleanupHourlyMetricsAPIView.as_view(),+ name="cleanup_hourly_metrics",+ ),+ path(+ "cleanup/daily/",+ internal_views.CleanupDailyMetricsAPIView.as_view(),+ name="cleanup_daily_metrics",+ ),+]diff --git a/backend/dashboard_metrics/internal_views.py b/backend/dashboard_metrics/internal_views.pynew file mode 100644index 000000000..f77663394--- /dev/null+++ b/backend/dashboard_metrics/internal_views.py@@ -0,0 +1,115 @@+"""Internal API for running the dashboard-metrics periodics (UN-3796).++Beat + ``workerMetrics`` are the last two Celery deployments blocking the+"scale every Celery deployment to zero" gate. The PG scheduler replaces Beat, but the+three ``dashboard_metrics.*`` tasks are **Django** — ORM, cache, a Redis lock — and the+PG consumer bootstraps its tasks from ``workers/`` with no Django, so it cannot run them+directly.++These endpoints are the execution half, following the pattern already used by+``ProcessLogHistoryAPIView`` / ``process_notification_buffer``: a thin worker-side task+POSTs here, and the backend runs the real function. The task bodies are plain functions+that happen to carry ``@shared_task``, so they are called **verbatim** — the aggregation+logic, its windows and its Redis lock are reused unchanged, not reimplemented.++Auth is entirely ``InternalAPIAuthMiddleware``, which fires on any ``/internal/`` path.++**Org context is deliberately absent.** The Celery path runs with no organization set —+which is why ``tasks.py`` uses ``_base_manager`` throughout — and the middleware+populates ``StateStore`` from ``X-Organization-ID``. The callers must not send that+header, and each view clears the slot defensively: ``StateStore`` is a thread-local and+gunicorn reuses threads, so a value left behind by an earlier request on the same thread+would silently scope these global aggregations to one tenant.+"""++import contextlib+import logging+from typing import Any++from rest_framework import status+from rest_framework.request import Request+from rest_framework.response import Response+from rest_framework.views import APIView+from utils.constants import Account+from utils.local_context import StateStore++from dashboard_metrics.tasks import (+ aggregate_metrics_from_sources,+ cleanup_daily_metrics,+ cleanup_hourly_metrics,+)++logger = logging.getLogger(__name__)++# Mirrors the defaults the Beat rows carry in their kwargs+# (dashboard_metrics/migrations/0002_setup_periodic_tasks.py), so a caller that omits+# them gets the same retention the Celery path applies.+DEFAULT_HOURLY_RETENTION_DAYS = 30+DEFAULT_DAILY_RETENTION_DAYS = 365+++def _clear_org_context() -> None:+ """Drop any organization left in this thread's StateStore. See the module docstring.++ ``suppress`` because ``StateStore.clear`` raises when the slot was never set, which+ is the normal case and not an error.+ """+ with contextlib.suppress(Exception):+ StateStore.clear(Account.ORGANIZATION_ID)+++def _int_arg(request: Request, key: str, default: int) -> int:+ """Read an optional positive integer from the request body."""+ raw = request.data.get(key, default) if isinstance(request.data, dict) else default+ try:+ value = int(raw)+ except (TypeError, ValueError) as exc:+ raise ValueError(f"{key} must be an integer, got {raw!r}") from exc+ if value < 1:+ raise ValueError(f"{key} must be >= 1, got {value}")+ return value+++class _MetricsTaskAPIView(APIView):+ """Shared plumbing: clear org context, run, translate errors."""++ def _run(self, fn, *args: Any, **kwargs: Any) -> Response:+ _clear_org_context()+ try:+ return Response(fn(*args, **kwargs))+ except ValueError as exc: # bad request body+ return Response({"error": str(exc)}, status=status.HTTP_400_BAD_REQUEST)+ except Exception as exc:+ logger.error("dashboard-metrics internal call failed: %s", exc, exc_info=True)+ return Response(+ {"error": str(exc)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR+ )+++class AggregateMetricsAPIView(_MetricsTaskAPIView):+ """Run the metrics aggregation.++ Calls the Celery task body verbatim, Redis lock included — this endpoint exists+ only because the PG consumer has no Django, not to change what the job does.+ """++ def post(self, request: Request) -> Response:+ return self._run(aggregate_metrics_from_sources)+++class CleanupHourlyMetricsAPIView(_MetricsTaskAPIView):+ def post(self, request: Request) -> Response:+ try:+ days = _int_arg(request, "retention_days", DEFAULT_HOURLY_RETENTION_DAYS)+ except ValueError as exc:+ return Response({"error": str(exc)}, status=status.HTTP_400_BAD_REQUEST)+ return self._run(cleanup_hourly_metrics, retention_days=days)+++class CleanupDailyMetricsAPIView(_MetricsTaskAPIView):+ def post(self, request: Request) -> Response:+ try:+ days = _int_arg(request, "retention_days", DEFAULT_DAILY_RETENTION_DAYS)+ except ValueError as exc:+ return Response({"error": str(exc)}, status=status.HTTP_400_BAD_REQUEST)+ return self._run(cleanup_daily_metrics, retention_days=days)diff --git a/backend/dashboard_metrics/migrations/0004_pg_periodic_tasks.py b/backend/dashboard_metrics/migrations/0004_pg_periodic_tasks.pynew file mode 100644index 000000000..463966a5b--- /dev/null+++ b/backend/dashboard_metrics/migrations/0004_pg_periodic_tasks.py@@ -0,0 +1,97 @@+"""Declare the dashboard-metrics periodics for the PG scheduler (UN-3796).++The PG twin of ``0002_setup_periodic_tasks``, which declares the same three schedules+for Celery Beat. **Deliberately in this app rather than in ``pg_queue``**: the failure+mode that matters is the two declarations drifting apart, and a reviewer editing a+schedule sees both only if they sit side by side.++Why a data migration here, when the pipeline mirror is a management command: these are+three fixed rows known at build time, so a migration is the right tool — tiny, idempotent+(``update_or_create``), and it reaches every environment including on-prem with no+operator step. Pipeline schedules are bulk and per-environment, so they stay a chunked+command that runs outside the migrate transaction.++Rows land **inert**: ``pg_owned=False`` and ``next_run_at=NULL``. Nothing fires from this+migration — the PG scheduler skips rows it does not own, and a NULL ``next_run_at`` means+"record a baseline next tick" rather than "overdue, fire now", so enabling the flag never+produces a burst of catch-up runs.++``task_kwargs`` is stored **decoded**: Beat keeps ``kwargs`` as a JSON *string*+(``'{"retention_days": 30}'``) while ``PgPeriodicTask.task_kwargs`` is a JSONField, so the+dispatcher can build a payload without re-parsing per tick.+"""++from django.db import migrations++# Single source for both directions, and importable by the drift test. Mirrors the Beat+# declarations in 0002_setup_periodic_tasks one-for-one — same names (the mirror key is+# PeriodicTask.name), same queue, same kwargs, and cron strings equivalent to the+# IntervalSchedule/CrontabSchedule rows there.+PG_PERIODIC_TASKS = [+ {+ "name": "dashboard_metrics_aggregate_from_sources",+ "task_name": "dashboard_metrics.aggregate_from_sources",+ "queue": "dashboard_metric_events",+ "task_args": [],+ "task_kwargs": {},+ # Beat: IntervalSchedule(every=15, period="minutes")+ "cron_string": "*/15 * * * *",+ },+ {+ "name": "dashboard_metrics_cleanup_hourly",+ "task_name": "dashboard_metrics.cleanup_hourly_data",+ "queue": "dashboard_metric_events",+ "task_args": [],+ "task_kwargs": {"retention_days": 30},+ # Beat: CrontabSchedule(minute=0, hour=2, every day) UTC+ "cron_string": "0 2 * * *",+ },+ {+ "name": "dashboard_metrics_cleanup_daily",+ "task_name": "dashboard_metrics.cleanup_daily_data",+ "queue": "dashboard_metric_events",+ "task_args": [],+ "task_kwargs": {"retention_days": 365},+ # Beat: CrontabSchedule(minute=0, hour=3, day_of_week=0 → Sunday) UTC+ "cron_string": "0 3 * * 0",+ },+]+++def create_pg_periodic_tasks(apps, schema_editor):+ PgPeriodicTask = apps.get_model("pg_queue", "PgPeriodicTask")+ for spec in PG_PERIODIC_TASKS:+ PgPeriodicTask.objects.update_or_create(+ name=spec["name"],+ defaults={+ "task_name": spec["task_name"],+ "queue": spec["queue"],+ "task_args": spec["task_args"],+ "task_kwargs": spec["task_kwargs"],+ "cron_string": spec["cron_string"],+ "org_id": "",+ "enabled": True,+ # Inert until the rollout flag decides otherwise; never fired by+ # applying this migration.+ "pg_owned": False,+ },+ )+++def remove_pg_periodic_tasks(apps, schema_editor):+ PgPeriodicTask = apps.get_model("pg_queue", "PgPeriodicTask")+ PgPeriodicTask.objects.filter(+ name__in=[spec["name"] for spec in PG_PERIODIC_TASKS]+ ).delete()+++class Migration(migrations.Migration):+ dependencies = [+ ("dashboard_metrics", "0003_alter_eventmetricsdaily_organization_and_more"),+ # The table this seeds.+ ("pg_queue", "0003_pgperiodictask"),+ ]++ operations = [+ migrations.RunPython(create_pg_periodic_tasks, remove_pg_periodic_tasks),+ ]diff --git a/backend/dashboard_metrics/tests/test_pg_periodic_task_declarations.py b/backend/dashboard_metrics/tests/test_pg_periodic_task_declarations.pynew file mode 100644index 000000000..85ea40789--- /dev/null+++ b/backend/dashboard_metrics/tests/test_pg_periodic_task_declarations.py@@ -0,0 +1,113 @@+"""Drift guard between the Beat and PG declarations of the metrics periodics (UN-3796).++Two migrations declare the same three schedules — ``0002_setup_periodic_tasks`` for Celery+Beat and ``0004_pg_periodic_tasks`` for the PG scheduler. They are separate rows in+separate tables, so nothing stops someone editing one and forgetting the other. That is+the whole failure mode this file exists for: a schedule changed on Beat but not on PG means+the task silently runs on a different cadence the moment the flag flips.++DB-free — both migration modules are imported and their declared specs compared directly,+so this runs in the unit tier rather than needing a migrated database.+"""++from __future__ import annotations++import importlib+import json++import pytest++_BEAT_MIGRATION = "dashboard_metrics.migrations.0002_setup_periodic_tasks"+_PG_MIGRATION = "dashboard_metrics.migrations.0004_pg_periodic_tasks"++# Cron equivalent of each Beat schedule, asserted against what the Beat migration builds.+# Written out rather than derived: deriving it from the same code under test would make+# the comparison vacuous.+_EXPECTED_CRON = {+ "dashboard_metrics_aggregate_from_sources": "*/15 * * * *",+ "dashboard_metrics_cleanup_hourly": "0 2 * * *",+ "dashboard_metrics_cleanup_daily": "0 3 * * 0",+}++[email protected](scope="module")+def pg_specs() -> dict[str, dict]:+ mod = importlib.import_module(_PG_MIGRATION)+ return {spec["name"]: spec for spec in mod.PG_PERIODIC_TASKS}+++class _FakeQuerySet:+ """Captures update_or_create calls from the Beat migration without a database."""++ def __init__(self, sink: dict):+ self._sink = sink++ def get_or_create(self, **kwargs):+ # Schedule rows (Interval/Crontab) — return the kwargs so the PeriodicTask+ # call can be inspected for which schedule it was given.+ return kwargs, True++ def update_or_create(self, name=None, defaults=None, **_kw):+ self._sink[name] = defaults or {}+ return defaults, True++ def filter(self, *_a, **_k):+ return self++ def delete(self):+ return (0, {})++[email protected](scope="module")+def beat_specs() -> dict[str, dict]:+ """Run the Beat migration's forward function against fakes and capture what it declares."""+ mod = importlib.import_module(_BEAT_MIGRATION)+ captured: dict[str, dict] = {}++ class _Apps:+ def get_model(self, _app, model):+ if model == "PeriodicTask":+ return type("PT", (), {"objects": _FakeQuerySet(captured)})+ return type("S", (), {"objects": _FakeQuerySet({})})++ mod.create_periodic_tasks(_Apps(), None)+ return captured+++class TestDeclarationsAgree:+ def test_same_set_of_schedules(self, beat_specs, pg_specs):+ # A schedule added to Beat but not PG stops firing the moment the flag flips;+ # the reverse fires something Beat never knew about.+ assert set(beat_specs) == set(pg_specs)++ @pytest.mark.parametrize("name", sorted(_EXPECTED_CRON))+ def test_task_path_and_queue_match(self, beat_specs, pg_specs, name):+ assert pg_specs[name]["task_name"] == beat_specs[name]["task"]+ assert pg_specs[name]["queue"] == beat_specs[name]["queue"]++ @pytest.mark.parametrize("name", sorted(_EXPECTED_CRON))+ def test_kwargs_match_once_decoded(self, beat_specs, pg_specs, name):+ # Beat stores kwargs as a JSON *string*; PgPeriodicTask.task_kwargs is a+ # JSONField. A mismatch here means the cleanup runs with the wrong retention.+ beat_kwargs = json.loads(beat_specs[name].get("kwargs") or "{}")+ assert pg_specs[name]["task_kwargs"] == beat_kwargs++ @pytest.mark.parametrize("name,cron", sorted(_EXPECTED_CRON.items()))+ def test_cron_matches_the_beat_cadence(self, pg_specs, name, cron):+ assert pg_specs[name]["cron_string"] == cron+++class TestSeededInert:+ """Applying the migration must not cause anything to fire."""++ def test_no_spec_declares_itself_pg_owned(self, pg_specs):+ # pg_owned is set to False in the migration's defaults, never from the spec —+ # this pins that no spec can smuggle ownership in.+ assert not any("pg_owned" in spec for spec in pg_specs.values())++ def test_no_spec_presets_a_run_time(self, pg_specs):+ # A non-NULL next_run_at in the past would read as "overdue" and fire a burst+ # of catch-up runs the moment the flag is enabled.+ for spec in pg_specs.values():+ assert "next_run_at" not in spec+ assert "last_run_at" not in specdiff --git a/backend/entrypoint.sh b/backend/entrypoint.shindex 20441c58f..9766b01ad 100755--- a/backend/entrypoint.sh+++ b/backend/entrypoint.sh@@ -27,6 +27,49 @@ doneif [ "$migrate" = true ]; thenecho "Migration initiated".venv/bin/python manage.py migrate++ # Converge schedule ownership to whatever PG_SCHEDULER_ENABLED declares (UN-3796)+ # — adopt when on, release back to Beat when off. NOT a migration: an ordinary+ # management command, sequenced after migrate only because pg_periodic_schedule+ # must exist first.+ #+ # READ THIS BEFORE ASSUMING A RESTART IS INERT. This is NOT the earlier+ # `--mirror-only` invocation, and it does NOT leave Beat alone: with+ # PG_SCHEDULER_ENABLED=true it flips pg_owned and DISABLES the Beat PeriodicTask+ # row for every mirrored pipeline THE pg_queue_enabled FLAG SELECTS — ownership is+ # additionally per-schedule Flipt-gated and fails closed, so with Flipt blind or the+ # flag at 0% this moves nothing and Beat keeps every row. A rolling restart therefore can move schedules+ # off Beat. (A comment here used to describe --mirror-only and state that+ # "ownership hand-over stays an explicit operator action" — it survived the switch+ # to converge_pg_scheduler and told an SRE auditing exactly this question the+ # opposite of the truth.)+ #+ # Running it on every start is what makes the ROLLBACK real: flipping the env var+ # back is enough, with no operator remembering a management command in every+ # environment (on-prem included, where `manage.py` is not something a deploy can+ # reach). Idempotent both ways in OUTCOME, but NOT free: with the gate on it+ # rewrites pg_periodic_schedule and the Beat PeriodicTask row and bumps+ # PeriodicTasks.update_changed() for every schedule, whether or not ownership+ # changed — roughly 2N row writes and N Beat reloads per pod start. (A previous+ # version of this line said "a restart that changes nothing writes nothing"; that+ # was wrong, and contradicted converge_pg_scheduler's own docstring.) The release+ # direction runs --mirror-only and skips the reconcile, so it costs nothing here.+ #+ # PG_SCHEDULER_ADOPT_PERIODICS additionally moves the dashboard_metrics.* rows;+ # it is separate because adopting them needs workerPgMetrics deployed.+ #+ # Best-effort by design: a convergence failure must never stop the backend from+ # starting. Note it is NOT all-or-nothing though — convergence commits per schedule+ # as it goes, so a failure can leave a partial hand-over. The command's summary line+ # names the failed count; re-run it or flip the env var back.+ PERIODICS_FLAG=""+ if [ "$(printf '%s' "${PG_SCHEDULER_ADOPT_PERIODICS:-}" | tr '[:upper:]' '[:lower:]')" = "true" ]; then+ PERIODICS_FLAG="--periodics"+ fi+ echo "PG schedule ownership convergence initiated"+ # shellcheck disable=SC2086 # intentional word-splitting: empty = flag absent+ .venv/bin/python manage.py converge_pg_scheduler $PERIODICS_FLAG \+ || echo "WARNING: PG schedule convergence failed; continuing startup (schedules keep their current firer)"fi# Configure Gunicorn based on --dev flag.diff --git a/backend/pg_queue/management/commands/converge_pg_scheduler.py b/backend/pg_queue/management/commands/converge_pg_scheduler.pynew file mode 100644index 000000000..a4341f582--- /dev/null+++ b/backend/pg_queue/management/commands/converge_pg_scheduler.py@@ -0,0 +1,115 @@+"""Converge schedule ownership to the state ``PG_SCHEDULER_ENABLED`` declares.++One idempotent entry point, **both directions**:++ PG_SCHEDULER_ENABLED=true → adopt (PG fires; Beat rows disabled)+ PG_SCHEDULER_ENABLED=false → release (Beat fires; pg_owned cleared)++Why one converging command rather than an adopt command and a release command: the+reverse direction is the one that matters under pressure, and a rollback nobody can+run is not a rollback. Splitting it left the reverse as a *procedure* — remember the+command, remember the flags, remember to run it in every environment. Here the env+var declares intent and the deploy makes the database match, so reverting is a values+change like any other, and re-running changes nothing once converged.++**Safe to run unattended**, which is what lets ``entrypoint.sh`` call it on every+start:++* Both directions are idempotent in OUTCOME — ``_set_ownership`` skips periodic rows+ already in the target state. The pipeline path is idempotent but **not** free: it+ rewrites ``pg_periodic_schedule`` and the Beat ``PeriodicTask`` row and bumps+ ``PeriodicTasks.update_changed()`` for every schedule on every call, whether or not+ ownership changed — so a backend start costs 2N row writes and N Beat reloads. Safe+ to repeat, not a no-op; an earlier version of this line claimed the latter.+* Neither direction invents state. Beat's ``PeriodicTask`` rows are only ever+ *disabled* and *re-enabled*, never created or deleted, and the value written on+ release is the one recorded before adoption (``pg_periodic_task.enabled``, and the+ pipeline's own ``active``) — so a schedule an operator had switched off stays off+ through a full adopt→release cycle.+* Adoption is not unilateral: it happens only because someone set the env var.++**What it cannot do, and you must:** converging to Beat restores *ownership*, not+*capacity*. Beat publishes to RabbitMQ, so a released schedule only fires again if+``workerSchedulerV2`` (and ``workerMetrics`` for the periodics) are running. Flip+those back in the SAME change that sets ``PG_SCHEDULER_ENABLED=false`` — otherwise+you have simply moved the outage. The release path logs a warning saying so.++Periodics (``dashboard_metrics.*``) are opt-in via ``--periodics``: pipelines and+metrics are separate rollout decisions, and metrics have their own consumer+(``workerPgMetrics``) that has to be deployed before they can be adopted.+"""++from typing import Any++from django.core.management import call_command+from django.core.management.base import BaseCommand+from scheduler.ownership import pg_scheduler_enabled+++class Command(BaseCommand):+ help = (+ "Converge schedule ownership to what PG_SCHEDULER_ENABLED declares: adopt to "+ "the PG scheduler when on, release back to Celery Beat when off. Idempotent "+ "in both directions and safe to run on every deploy."+ )++ def add_arguments(self, parser: Any) -> None:+ parser.add_argument(+ "--dry-run",+ action="store_true",+ help="Report what would change without writing.",+ )+ parser.add_argument(+ "--periodics",+ action="store_true",+ help=(+ "Also converge the non-pipeline periodics (dashboard_metrics.*). "+ "Off by default: adopting them requires workerPgMetrics to be "+ "deployed, so it is a separate rollout decision from pipelines."+ ),+ )++ def handle(self, *args: Any, **options: Any) -> None:+ dry_run = options["dry_run"]+ periodics = options["periodics"]+ target_pg = pg_scheduler_enabled()++ self.stdout.write(+ self.style.MIGRATE_HEADING(+ f"Converging schedule ownership → "+ f"{'PG scheduler' if target_pg else 'Celery Beat'} "+ f"(PG_SCHEDULER_ENABLED={'true' if target_pg else 'false'})"+ )+ )++ if target_pg:+ # Mirrors are backfilled first: a pipeline with no mirror row cannot be+ # owned, and reconcile reports it as still-on-Beat rather than adopting it.+ call_command("reconcile_pg_schedules", dry_run=dry_run)+ if periodics:+ # An empty name list means "every mirrored row" (the flag is+ # `nargs="*"`, and the command distinguishes absent from empty).+ # mirror_pg_periodic_tasks always backfills before flipping+ # ownership, so this one call covers both halves.+ call_command("mirror_pg_periodic_tasks", adopt=[], dry_run=dry_run)+ else:+ # --mirror-only keeps the backfill (inert at any flag state) while+ # --release-stale hands back anything still marked pg_owned. Together+ # they are the pipeline rollback.+ call_command(+ "reconcile_pg_schedules",+ mirror_only=True,+ release_stale=True,+ dry_run=dry_run,+ )+ if periodics:+ call_command("mirror_pg_periodic_tasks", release=[], dry_run=dry_run)+ self.stdout.write(+ self.style.WARNING(+ "Released to Beat. Beat publishes over RabbitMQ — these schedules "+ "fire again ONLY if workerSchedulerV2 (and workerMetrics, with "+ "--periodics) are running. Check that before relying on this."+ )+ )++ self.stdout.write(self.style.SUCCESS("Convergence complete."))diff --git a/backend/pg_queue/management/commands/mirror_pg_periodic_tasks.py b/backend/pg_queue/management/commands/mirror_pg_periodic_tasks.pynew file mode 100644index 000000000..7073d98c3--- /dev/null+++ b/backend/pg_queue/management/commands/mirror_pg_periodic_tasks.py@@ -0,0 +1,427 @@+"""Mirror non-pipeline Beat periodics into ``pg_periodic_task``, and hand them over.++The Beat-replacement half for everything that is not a scheduled pipeline+(UN-3796): ``dashboard_metrics.*``, log-history, audit, and anything an operator+has added. Pipeline schedules keep their own mirror and their own percentage ramp+(``reconcile_pg_schedules``); this command deliberately skips them.++Two steps, deliberately separate:++* **mirror** (default) — upsert a ``PgPeriodicTask`` row for every non-pipeline+ ``PeriodicTask``. Purely additive and inert: rows land ``pg_owned=False``, so the+ PG scheduler still fires nothing and Beat keeps firing everything.+* **adopt / release** (explicit flags) — the actual hand-over. ``--adopt`` flips+ ``pg_owned=True`` **and** disables the matching Beat ``PeriodicTask``, in one+ transaction. ``--release`` reverses it. Doing both halves atomically is the whole+ point: a row that is ``pg_owned`` while Beat still has it enabled fires **twice**,+ which for ``cleanup_*`` means two concurrent deletes and for ``aggregate_*`` means+ double-counted metrics.++Idempotent and safe to re-run. Mirroring changes no behaviour on its own; only+``--adopt`` does, and only for the rows it names.++Unlike the pipeline ramp there is no percentage: these are a handful of global+singletons, and the acceptance gate (Celery scaled to zero) needs all of them on PG,+so the meaningful states are "all Beat" and "all PG" with a per-name escape hatch.+"""++import json+import logging+from typing import Any, NamedTuple++from django.core.management.base import BaseCommand, CommandError+from django.db import transaction+from django.utils import timezone+from django_celery_beat.models import IntervalSchedule, PeriodicTask, PeriodicTasks++from pg_queue.models import PgPeriodicTask++logger = logging.getLogger(__name__)++# Rows per DB round trip. Matches the batch size the repo's other bounded loops use+# (e.g. workflow_v2 migration 0012's "delete in batches to avoid long-running+# transactions"). Overridable with --batch-size.+DEFAULT_BATCH_SIZE = 1000++# Task paths this command must NOT mirror.+#+# Pipeline triggers own a separate mirror (pg_periodic_schedule) and a separate+# percentage ramp; mirroring one here too would give it two owners. BOTH known task+# paths are listed even though ``SchedulerHelper._schedule_task_job`` only writes the+# first today: a real Beat table was found carrying a legacy ``execute_pipeline_task_v2``+# row, and it is only excluded here by luck of being disabled. Matching on the task+# path rather than a name convention keeps that from depending on luck.+#+# ``celery.backend_cleanup`` is Celery's own built-in: it prunes the Celery RESULT+# BACKEND. It is meaningless on PG (nothing there registers it, and the backend it+# cleans stops existing once Celery is off), so it stays with Beat and simply retires+# alongside it.+_EXCLUDED_TASK_PATHS = frozenset(+ {+ "scheduler.tasks.execute_pipeline_task",+ "execute_pipeline_task_v2",+ "celery.backend_cleanup",+ }+)+++def cron_from_periodic_task(task: PeriodicTask) -> str:+ """Best-effort 5-field cron for a Beat periodic, or "" if not expressible.++ Beat schedules a task by ``crontab``, ``interval``, ``solar`` or ``clocked``.+ Only the first two are in use here, and only they map onto cron:++ * ``crontab`` — a direct field-for-field reconstruction.+ * ``interval`` — expressed as a step cron ONLY where one exists exactly.++ Returns ``""`` for anything else — notably **second**-resolution intervals,+ which have no cron expression at all. Coarsening one to a minute would silently+ change how often it runs, so the caller skips those and says so rather than+ guessing.++ **``*/N`` is not "every N".** A step cron restarts at each field boundary, so it+ is faithful only when N divides the field's range exactly. Otherwise the last+ step of one period runs into the first of the next and the task fires early —+ silently, and only at the boundary, which is the hardest kind of drift to spot:++ * ``*/7`` minutes → :00 :07 … :56, then **:00** — a 4-minute gap, not 7.+ * ``*/45`` minutes → :00 :45, then **:00** — fires roughly twice as often.+ * ``0 */5`` hours → 0 5 10 15 20, then **0** — a 4-hour gap, not 5.++ So minutes need ``60 % every == 0`` and hours ``24 % every == 0``, not merely a+ range check.++ **Days are worse and are refused beyond 1.** ``*/N`` on day-of-month restarts+ every month, and months are 28-31 days, so the gap at the boundary varies by+ month and even by year: ``0 0 */7 * *`` fires on the 1st, 8th, 15th, 22nd, 29th+ and then the 1st again — 2 to 4 days later depending on the month. There is no+ correct cron for "every N days" at N > 1, so only ``every == 1`` maps (to a+ plain daily), and the rest fall through to the caller's skip-and-explain path.++ An earlier version range-checked (``every < 60`` / ``< 24`` / ``< 32``) while+ the docstring claimed exactness, so an "every 45 minutes" periodic mirrored to+ something that fires twice as often. Nothing in integration hit it (15 divides+ 60), but Beat schedules are per-environment DB rows that exist in no source+ file, so staging or production can carry one.+ """+ if task.crontab is not None:+ c = task.crontab+ return f"{c.minute} {c.hour} {c.day_of_month} {c.month_of_year} {c.day_of_week}"+ interval = task.interval+ if interval is None:+ return ""+ every = interval.every+ if every < 1:+ return ""+ if interval.period == IntervalSchedule.MINUTES and every < 60 and 60 % every == 0:+ return f"*/{every} * * * *"+ if interval.period == IntervalSchedule.HOURS and every < 24 and 24 % every == 0:+ return f"0 */{every} * * *"+ if interval.period == IntervalSchedule.DAYS and every == 1:+ return "0 0 * * *"+ # SECONDS, a step that does not divide its field, or anything else.+ return ""+++def _decode(raw: str | None, fallback: Any) -> Any:+ """Beat stores args/kwargs as JSON text; decode once here so a malformed value+ fails at mirror time (visible, fixable) instead of at fire time (a periodic+ that silently stops running).+ """+ if not raw:+ return fallback+ return json.loads(raw)+++class MirrorPlan(NamedTuple):+ """What to do with one Beat periodic: mirror it, or skip it and say why.++ Every decision this command makes lives here rather than inside the loop that+ talks to the database, so the rules can be tested against plain stand-ins+ instead of a live multi-tenant schema. The command becomes glue: plan, then+ apply.+ """++ name: str+ fields: dict[str, Any] | None # None => skip+ skip_reason: str | None = None++ @property+ def should_mirror(self) -> bool:+ return self.fields is not None+++def plan_mirror(task: Any) -> MirrorPlan:+ """Decide whether one Beat periodic can be mirrored, and with what fields.++ Accepts anything exposing the ``PeriodicTask`` attributes used here, so the+ rules are testable without the ORM.+ """+ if task.task in _EXCLUDED_TASK_PATHS:+ return MirrorPlan(+ task.name,+ None,+ f"{task.task} is owned elsewhere (pipeline mirror or Celery-internal)",+ )+ cron = cron_from_periodic_task(task)+ if not cron:+ return MirrorPlan(+ task.name,+ None,+ "schedule has no cron equivalent (second-resolution interval, solar "+ "or clocked) — it must stay on Beat or move to a different mechanism",+ )+ try:+ task_args = _decode(task.args, [])+ task_kwargs = _decode(task.kwargs, {})+ except ValueError as exc:+ return MirrorPlan(task.name, None, f"malformed args/kwargs JSON ({exc})")+ return MirrorPlan(+ task.name,+ {+ "task_name": task.task,+ # Beat falls back to the default queue when unset; mirror the same+ # fallback so the row targets where Beat would have.+ "queue": task.queue or "celery",+ "task_args": task_args,+ "task_kwargs": task_kwargs,+ "cron_string": cron,+ "enabled": task.enabled,+ },+ )+++class Command(BaseCommand):+ help = (+ "Mirror non-pipeline Beat periodics into pg_periodic_task. Additive and "+ "inert by default; --adopt hands rows over to PG (and disables them in "+ "Beat) atomically, --release reverses it."+ )++ def add_arguments(self, parser: Any) -> None:+ parser.add_argument(+ "--dry-run",+ action="store_true",+ help="Report what would change without writing.",+ )+ parser.add_argument(+ "--adopt",+ nargs="*",+ metavar="NAME",+ help=(+ "Hand rows over to PG: set pg_owned=True and DISABLE the matching "+ "Beat PeriodicTask, atomically. Pass names to adopt those only, or "+ "no names to adopt every mirrored row."+ ),+ )+ parser.add_argument(+ "--release",+ nargs="*",+ metavar="NAME",+ help="Reverse of --adopt: pg_owned=False and re-enable the Beat task.",+ )+ parser.add_argument(+ "--batch-size",+ type=int,+ default=DEFAULT_BATCH_SIZE,+ metavar="N",+ help=(+ f"Rows fetched per DB round trip (default {DEFAULT_BATCH_SIZE}). "+ "Bounds memory on a large Beat table."+ ),+ )+ parser.add_argument(+ "--limit",+ type=int,+ default=0,+ metavar="N",+ help=(+ "Stop after mirroring N rows (0 = no limit). A safety valve for a "+ "first cautious run, NOT a resume cursor: the scan always starts "+ "from the beginning, so re-running with --limit re-visits the same "+ "rows. Drop it to mirror everything."+ ),+ )++ def handle(self, *args: Any, **options: Any) -> None:+ if options["adopt"] is not None and options["release"] is not None:+ raise CommandError("--adopt and --release are mutually exclusive")+ if options["batch_size"] < 1:+ raise CommandError("--batch-size must be >= 1")+ if options["limit"] < 0:+ raise CommandError("--limit must be >= 0")+ dry_run = options["dry_run"]+ mirrored, skipped = self._mirror(+ dry_run, batch_size=options["batch_size"], limit=options["limit"]+ )++ moved = 0+ if options["adopt"] is not None:+ moved = self._set_ownership(+ options["adopt"],+ to_pg=True,+ dry_run=dry_run,+ batch_size=options["batch_size"],+ )+ elif options["release"] is not None:+ moved = self._set_ownership(+ options["release"],+ to_pg=False,+ dry_run=dry_run,+ batch_size=options["batch_size"],+ )++ prefix = "[dry-run] " if dry_run else ""+ summary = (+ f"{prefix}mirrored={mirrored} skipped={skipped} ownership_changed={moved}"+ )+ if skipped:+ # Not fatal, but it means Beat is still the only firer for those — which+ # blocks scaling Beat to zero. Surface it where the operator looks.+ self.stderr.write(self.style.WARNING(summary))+ else:+ self.stdout.write(self.style.SUCCESS(summary))++ def _mirror(self, dry_run: bool, *, batch_size: int, limit: int) -> tuple[int, int]:+ """Apply :func:`plan_mirror` to every Beat periodic. Never changes ownership.++ Deliberately thin: every rule lives in ``plan_mirror``; this only reports and+ writes. A skip is loud because it means Beat remains the sole firer for that+ row, which is what blocks scaling Beat to zero.++ Scale: the excluded task paths are filtered **in SQL**, not in Python. There is+ one ``PeriodicTask`` row per scheduled pipeline, so matching them in Python+ would drag the entire pipeline population through memory only to discard it.+ ``.iterator()`` then bounds what is held at once. ``--limit`` caps a single+ run for a cautious first pass; it is not a resume cursor, since the upsert is+ idempotent and the scan always restarts from the beginning.+ """+ mirrored = skipped = 0+ # order_by(pk) makes the scan deterministic, so a --limit run and its+ # follow-ups walk the table in a stable order rather than re-treading rows.+ queryset = (+ PeriodicTask.objects.exclude(task__in=_EXCLUDED_TASK_PATHS)+ .select_related("crontab", "interval")+ .order_by("pk")+ )+ for task in queryset.iterator(chunk_size=batch_size):+ if limit and mirrored >= limit:+ self.stdout.write(+ f"--limit {limit} reached; {queryset.count() - mirrored} row(s) "+ "not visited. Re-run WITHOUT --limit to mirror everything "+ "(the scan restarts from the beginning either way)."+ )+ break+ plan = plan_mirror(task)+ if not plan.should_mirror:+ # Excluded paths never reach here (filtered in SQL above), so every+ # skip is something an operator must resolve before Beat can go away.+ skipped += 1+ self.stderr.write(+ self.style.WARNING(f"skipping {plan.name!r}: {plan.skip_reason}")+ )+ continue+ self.stdout.write(+ f"mirror {plan.name!r} task={plan.fields['task_name']} "+ f"queue={plan.fields['queue']!r} cron={plan.fields['cron_string']!r} "+ f"enabled={plan.fields['enabled']}"+ )+ if not dry_run:+ fields = dict(plan.fields)+ # `enabled` is the ONE mirrored field that stops tracking Beat once a+ # row is adopted: after --adopt Beat's copy is False by definition, so+ # re-mirroring would copy that back and leave pg_owned=True with+ # enabled=False — matching NEITHER firer (the PG tick selects+ # `WHERE pg_owned AND enabled`, Beat's row is disabled). The periodic+ # would stop silently, and the pre-migration value needed to release it+ # would be gone. Cron/args/queue still track Beat, so a schedule edit+ # made while PG owns the row is still picked up.+ if PgPeriodicTask.objects.filter(name=plan.name, pg_owned=True).exists():+ fields.pop("enabled", None)+ PgPeriodicTask.objects.update_or_create(name=plan.name, defaults=fields)+ mirrored += 1+ return mirrored, skipped++ def _set_ownership(+ self, names: list[str], *, to_pg: bool, dry_run: bool, batch_size: int+ ) -> int:+ """Flip pg_owned and the matching Beat task's enabled flag together.++ One transaction per row: the two halves must not be separable, since the+ window between them is exactly the double-fire (or no-fire) window.+ """+ rows = PgPeriodicTask.objects.all().order_by("pk")+ if names:+ rows = rows.filter(name__in=names)+ missing = set(names) - set(rows.values_list("name", flat=True))+ if missing:+ raise CommandError(+ f"no mirror row for: {', '.join(sorted(missing))} "+ f"(run without --adopt/--release first to mirror them)"+ )+ changed = 0+ # Deliberately NOT limited by --limit: an ownership flip is the operator's+ # explicit, named intent, and stopping half way would leave some rows on PG+ # and some on Beat with no record of where the boundary fell. It is chunked+ # only to bound memory. In practice this table holds a handful of rows.+ for row in rows.iterator(chunk_size=batch_size):+ if row.pg_owned == to_pg:+ continue+ # RESTORE, don't blanket-enable. On release this used to write+ # `enabled=True` unconditionally, which RESURRECTS a periodic an operator+ # had deliberately switched off in Beat before the migration — a rollback+ # is supposed to restore the previous state, not invent a new one.+ # `row.enabled` is Beat's own value, captured by the mirror (plan_mirror+ # copies task.enabled), so a row that was off mirrors off, is adopted off,+ # and is released off. On adopt the answer is always False: PG owns it.+ beat_enabled = False if to_pg else row.enabled+ verb = "adopt" if to_pg else "release"+ self.stdout.write(f"{verb} {row.name!r} (beat enabled -> {beat_enabled})")+ if not dry_run:+ with transaction.atomic():+ row.pg_owned = to_pg+ # Clear the baseline on release so a later re-adopt records a+ # fresh next_run_at instead of firing immediately for a+ # next_run_at that went stale while Beat owned the schedule.+ if not to_pg:+ row.next_run_at = None+ row.save(update_fields=["pg_owned", "next_run_at", "updated_at"])+ beat_updates: dict = {"enabled": beat_enabled}+ # Baseline Beat's clock on release, the mirror of clearing+ # next_run_at above. DatabaseScheduler derives due-ness from+ # PeriodicTask.last_run_at against the crontab, so a periodic that+ # spent days PG-owned is overdue by every interval it missed and+ # Beat replays them the moment `enabled` flips back. Seen on+ # integration 2026-08-24: all three dashboard_metrics.* fired+ # inside 30 ms of the release, alongside four pipelines.+ #+ # Release-only: on adopt Beat is being switched OFF, so its clock+ # is irrelevant, and touching it would corrupt the value a later+ # release needs to restore.+ if not to_pg:+ beat_updates["last_run_at"] = timezone.now()+ matched = PeriodicTask.objects.filter(name=row.name).update(+ **beat_updates+ )+ if not matched and not to_pg:+ # Release only. A bulk .update() returning 0 is indistinguishable+ # from success, and on release it means a periodic PG has let go+ # of that Beat cannot pick up — no firer at all, on the rollback+ # path. On ADOPT a missing Beat row just means there was nothing+ # to disable, which is benign; warning on both is how the real+ # one gets ignored.+ self.stderr.write(+ f"WARNING: no Beat PeriodicTask named {row.name!r} to "+ f"release to — it may now have no firer"+ )+ # Bulk .update() bypasses django-celery-beat's post_save signal,+ # so PeriodicTasks.last_update never bumps and DatabaseScheduler+ # never reloads. Without this, --adopt would set pg_owned=True and+ # flip the DB row to disabled while Beat kept firing it from its+ # stale in-memory copy — a DOUBLE FIRE, precisely what doing both+ # halves in one transaction exists to prevent. --release has the+ # mirror failure: Beat would never resume. Same fix and same+ # reason as scheduler/ownership.py:132 on the pipeline path.+ PeriodicTasks.update_changed()+ changed += 1+ return changeddiff --git a/backend/pg_queue/management/commands/reconcile_pg_schedules.py b/backend/pg_queue/management/commands/reconcile_pg_schedules.pyindex deaf059ee..f5e9cd6f4 100644--- a/backend/pg_queue/management/commands/reconcile_pg_schedules.py+++ b/backend/pg_queue/management/commands/reconcile_pg_schedules.py@@ -18,11 +18,19 @@ from typing import Anyfrom django.core.management.base import BaseCommand, CommandErrorfrom django_celery_beat.models import CrontabSchedule, PeriodicTask-from scheduler.ownership import reconcile_ownership_for, resolve_schedule_owner+from scheduler.ownership import (+ pg_scheduler_enabled,+ reconcile_ownership_for,+ resolve_schedule_owner,+)from scheduler.tasks import mirror_periodic_schedule_upsertfrom pg_queue.models import PgPeriodicSchedule+# Rows per DB round trip; mirrors mirror_pg_periodic_tasks.DEFAULT_BATCH_SIZE and the+# batch size used by the repo's other bounded loops (workflow_v2 migration 0012).+DEFAULT_BATCH_SIZE = 1000+# Only the pipeline-trigger PeriodicTasks are scheduled pipelines (other periodic# tasks — metrics, audit — are not mirrored)._PIPELINE_TASK_PATH = "scheduler.tasks.execute_pipeline_task"@@ -51,17 +59,75 @@ class Command(BaseCommand):action="store_true",help="Report what would change without writing.",)+ parser.add_argument(+ "--batch-size",+ type=int,+ default=DEFAULT_BATCH_SIZE,+ metavar="N",+ help=(+ f"Rows fetched per DB round trip (default {DEFAULT_BATCH_SIZE}). "+ "There is one row per scheduled pipeline, so this bounds memory on "+ "a large installation."+ ),+ )+ parser.add_argument(+ "--mirror-only",+ action="store_true",+ help=(+ "Backfill missing mirror rows and skip the ownership reconcile. "+ "Purely additive: touches only pg_periodic_schedule, never a Beat "+ "PeriodicTask. This is the mode automation runs ONLY on the release path (converge_pg_scheduler with PG_SCHEDULER_ENABLED=false); the adopt path runs the full reconcile — see handle()."+ ),+ )+ parser.add_argument(+ "--release-stale",+ action="store_true",+ help=(+ "Release schedules marked pg_owned while PG_SCHEDULER_ENABLED is "+ "off, handing them back to Beat. Safe for unattended automation at "+ "any flag state: with the gate ON it is a no-op, and with it off it "+ "only ever moves a schedule TO Beat — the fail-safe direction. "+ "Composes with --mirror-only."+ ),+ )def handle(self, *args: Any, **options: Any) -> None:dry_run = options["dry_run"]- backfilled = self._backfill_mirrors(dry_run)- reconciled, pg_owned, failed = self._reconcile_all(dry_run)+ batch_size = options["batch_size"]+ mirror_only = options["mirror_only"]+ if batch_size < 1:+ raise CommandError("--batch-size must be >= 1")+ backfilled = self._backfill_mirrors(dry_run, batch_size)++ # --mirror-only exists because it is safe to run at ANY flag state: backfilling+ # is inert, while the reconcile below — fail-closed when the rollout is off —+ # flips ownership *and disables the matching Beat PeriodicTask* when it is on.+ #+ # It is NO LONGER what the deploy-time automation runs. entrypoint.sh invokes+ # converge_pg_scheduler on every backend start, without --mirror-only, so+ # ownership hand-over IS an unattended action now; the env var is the operator's+ # consent, given once, rather than a command typed per environment. That is+ # deliberate — it is what makes rollback a values change — but it means this+ # path's caller is no longer the only way ownership moves. A previous version of+ # this comment ended "ownership stays an operator action", which stopped being+ # true when the entrypoint switched commands.+ if mirror_only:+ reconciled, pg_owned, failed = 0, 0, 0+ else:+ reconciled, pg_owned, failed = self._reconcile_all(dry_run, batch_size)++ released = 0+ if options["release_stale"]:+ released, release_failed = self._release_stale(dry_run, batch_size)+ failed += release_failedprefix = "[dry-run] " if dry_run else ""summary = (f"{prefix}backfilled={backfilled} reconciled={reconciled} "- f"pg_owned={pg_owned} failed={failed}"+ f"pg_owned={pg_owned} released={released} failed={failed}")+ if mirror_only:+ summary = f"{summary} (mirror-only: ownership reconcile skipped)"if failed:# Surface failures where the operator looks (and to automation).self.stderr.write(self.style.ERROR(summary))@@ -93,15 +159,30 @@ class Command(BaseCommand):"pipeline_name": task_args[6] if len(task_args) > 6 else "",}- def _backfill_mirrors(self, dry_run: bool) -> int:- """Create a mirror row for every pipeline-trigger PeriodicTask lacking one."""- # Pre-fetch the already-mirrored ids in one query (avoid an EXISTS per row).+ def _backfill_mirrors(self, dry_run: bool, batch_size: int) -> int:+ """Create a mirror row for every pipeline-trigger PeriodicTask lacking one.++ Both reads are bounded: the already-mirrored ids stream in via ``iterator``+ rather than materialising the whole table as a Python set, and the+ PeriodicTask scan is chunked. There is one row per scheduled pipeline, so on+ a large installation the unbounded version held the entire pipeline+ population in memory twice.+ """+ # Still one query, still an id set (the membership test below needs it), but+ # streamed and values-only — flat ids, never model instances.mirrored = {str(pk)- for pk in PgPeriodicSchedule.objects.values_list("pipeline_id", flat=True)+ for pk in PgPeriodicSchedule.objects.values_list(+ "pipeline_id", flat=True+ ).iterator(chunk_size=batch_size)}backfilled = 0- for pt in PeriodicTask.objects.filter(task=_PIPELINE_TASK_PATH):+ periodic_tasks = (+ PeriodicTask.objects.filter(task=_PIPELINE_TASK_PATH)+ .select_related("crontab")+ .order_by("pk")+ )+ for pt in periodic_tasks.iterator(chunk_size=batch_size):pipeline_id = pt.name # = str(pipeline.pk)if pipeline_id in mirrored:continue@@ -121,12 +202,60 @@ class Command(BaseCommand):backfilled += 1return backfilled- def _reconcile_all(self, dry_run: bool) -> tuple[int, int, int]:+ def _release_stale(self, dry_run: bool, batch_size: int) -> tuple[int, int]:+ """Hand every stale pg_owned row back to Beat. Returns (released, failed).++ Scoped to ``pg_owned=True`` rows so a clean installation does no work at all,+ and gated on the env switch being OFF: with the ramp ON, a pg_owned row is+ legitimate and releasing it would silently undo the rollout.++ Unlike :meth:`_reconcile_all` this IS safe to run unattended, because its only+ possible effect is moving a schedule to Beat — the same direction the system+ already fails to. That is what lets the deploy run it; see entrypoint.sh.+ """+ if pg_scheduler_enabled():+ self.stdout.write(+ "--release-stale: PG_SCHEDULER_ENABLED is on; pg_owned rows are "+ "legitimate here, nothing released."+ )+ return 0, 0++ released = failed = 0+ for row in (+ PgPeriodicSchedule.objects.filter(pg_owned=True)+ .order_by("pk")+ .iterator(chunk_size=batch_size)+ ):+ if dry_run:+ released += 1+ self.stdout.write(+ f"[dry-run] would release pipeline {row.pipeline_id} "+ f"({row.pipeline_name or 'unnamed'}) back to Beat"+ )+ continue+ # Routes through the same transaction the gate-off repair path uses, so+ # Beat's PeriodicTask is re-enabled and next_run_at cleared in step.+ result = reconcile_ownership_for(+ str(row.pipeline_id), row.organization_id, active=row.enabled+ )+ if result is None: # transaction failed (already logged)+ failed += 1+ continue+ released += 1+ return released, failed++ def _reconcile_all(self, dry_run: bool, batch_size: int) -> tuple[int, int, int]:"""Reconcile ownership for every mirror row against the current rollout.Returns (reconciled, pg_owned, failed).++ Chunked: this loads full model instances, one per scheduled pipeline, and+ each iteration does a Flipt evaluation plus a write — so it is the longest+ loop in the command and the one worth bounding."""reconciled = pg_owned = failed = 0- for row in PgPeriodicSchedule.objects.all():+ for row in PgPeriodicSchedule.objects.order_by("pk").iterator(+ chunk_size=batch_size+ ):if dry_run:# Preview only — read the would-be owner (no DB write) so an# operator can see how many a ramp change would hand to PG.diff --git a/backend/pg_queue/migrations/0002_pgqueuemessage_available_at.py b/backend/pg_queue/migrations/0002_pgqueuemessage_available_at.pynew file mode 100644index 000000000..d3eed988d--- /dev/null+++ b/backend/pg_queue/migrations/0002_pgqueuemessage_available_at.py@@ -0,0 +1,67 @@+"""Delayed visibility for pg_queue_message (UN-3843).++Adds ``available_at`` + the ``scheduled`` state so a dispatch can defer delivery+(Celery ``countdown``/``eta`` parity). Additive and inert: existing rows and every+existing enqueue resolve to ``available_at = now()`` / ``state = 'ready'``, which is+exactly today's behaviour.++**Why the hand-written ``SET DEFAULT now()`` step.** Django's ``AddField`` adds the+column with a one-off literal default and then DROPS that default, leaving the column+``NOT NULL`` with no DB default. The workers' enqueue is raw SQL with an explicit+column list that does not mention ``available_at``+(``queue_backend/pg_queue/client.py``) — so without a persistent DB default, every+worker enqueue would fail with a not-null violation the moment this migration landed,+and would keep failing regardless of deploy order. The DB default also encodes the+right semantic on its own: a row that does not ask to be deferred is available now.++Ordering is therefore safe in both directions (migrate-before-workers or+workers-before-migrate), which is the posture the rest of the PG rollout assumes.+"""++import django.utils.timezone+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("pg_queue", "0001_initial_squashed"),+ ]++ operations = [+ # Widen the closed state enum BEFORE anything can write 'scheduled'.+ migrations.RemoveConstraint(+ model_name="pgqueuemessage",+ name="pg_queue_message_state_valid",+ ),+ migrations.AddField(+ model_name="pgqueuemessage",+ name="available_at",+ field=models.DateTimeField(default=django.utils.timezone.now),+ ),+ # Restore a PERSISTENT default so the workers' raw INSERT (which omits this+ # column) keeps working. Django dropped the AddField default above; state_+ # operations is empty because this changes only the DB, not the model Django+ # tracks — the model keeps its Python-level default and stays in sync.+ migrations.RunSQL(+ sql="ALTER TABLE pg_queue_message ALTER COLUMN available_at SET DEFAULT now()",+ reverse_sql=(+ "ALTER TABLE pg_queue_message ALTER COLUMN available_at DROP DEFAULT"+ ),+ state_operations=[],+ ),+ migrations.AddIndex(+ model_name="pgqueuemessage",+ index=models.Index(+ models.F("available_at"),+ condition=models.Q(("state", "scheduled")),+ name="pg_queue_message_scheduled_idx",+ ),+ ),+ migrations.AddConstraint(+ model_name="pgqueuemessage",+ constraint=models.CheckConstraint(+ check=models.Q(("state__in", ["ready", "claimed", "scheduled"])),+ name="pg_queue_message_state_valid",+ ),+ ),+ ]diff --git a/backend/pg_queue/migrations/0003_pgperiodictask.py b/backend/pg_queue/migrations/0003_pgperiodictask.pynew file mode 100644index 000000000..cbeef73c9--- /dev/null+++ b/backend/pg_queue/migrations/0003_pgperiodictask.py@@ -0,0 +1,40 @@+# Generated by Django 4.2.30 on 2026-08-05 08:32++import django.utils.timezone+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("pg_queue", "0002_pgqueuemessage_available_at"),+ ]++ operations = [+ migrations.CreateModel(+ name="PgPeriodicTask",+ fields=[+ ("name", models.TextField(primary_key=True, serialize=False)),+ ("task_name", models.TextField()),+ ("queue", models.TextField()),+ ("task_args", models.JSONField(default=list)),+ ("task_kwargs", models.JSONField(default=dict)),+ ("cron_string", models.TextField()),+ ("org_id", models.TextField(blank=True, default="")),+ ("enabled", models.BooleanField(default=True)),+ ("pg_owned", models.BooleanField(default=False)),+ ("last_run_at", models.DateTimeField(blank=True, null=True)),+ ("next_run_at", models.DateTimeField(blank=True, null=True)),+ ("created_at", models.DateTimeField(default=django.utils.timezone.now)),+ ("updated_at", models.DateTimeField(auto_now=True)),+ ],+ options={+ "db_table": "pg_periodic_task",+ "indexes": [+ models.Index(+ fields=["pg_owned", "enabled", "next_run_at"],+ name="pg_periodic_task_due_idx",+ )+ ],+ },+ ),+ ]diff --git a/backend/pg_queue/models.py b/backend/pg_queue/models.pyindex abd5bab44..500c6d908 100644--- a/backend/pg_queue/models.py+++ b/backend/pg_queue/models.py@@ -8,6 +8,7 @@ from unstract.core.data_models import QueueMessageState# shared with the workers' raw SQL (client.py / reaper.py). See QueueMessageState._READY = QueueMessageState.READY.value_CLAIMED = QueueMessageState.CLAIMED.value+_SCHEDULED = QueueMessageState.SCHEDULED.valueclass PgQueueMessage(models.Model):@@ -71,6 +72,24 @@ class PgQueueMessage(models.Model):# aggressive autovacuum) — kept out of the migration so the schema stays# portable across Postgres providers.state = models.TextField(default=_READY)+ # Delayed visibility (UN-3843) — Celery ``countdown``/``eta`` parity. Set to a+ # FUTURE timestamp only together with ``state='scheduled'``; the pair is what+ # defers the row. Default now() keeps every pre-existing dispatch byte-identical+ # (immediate rows are enqueued 'ready' and this column is never read for them).+ #+ # Deliberately NOT a claim predicate. The claim's partial index holds only+ # 'ready' rows, so a scheduled row is physically absent from it and a pending+ # delay costs the hot path nothing. Adding `available_at <= now()` to the claim+ # would instead park every not-yet-due row inside the claim index to be walked+ # and discarded on every claim — reintroducing the O(in-flight) scan-past cost+ # that the state machine above was introduced to remove. The reaper promotes+ # 'scheduled' -> 'ready' when due (reaper.promote_due_scheduled), so delivery is+ # "not before available_at", never early, at reaper-tick granularity.+ #+ # Like `vt`, this column is absent from the claim index, so it never burdens the+ # hot path; unlike `vt` it is written exactly once (at enqueue) and then only+ # read by the promotion sweep.+ available_at = models.DateTimeField(default=timezone.now)class Meta:db_table = "pg_queue_message"@@ -82,7 +101,7 @@ class PgQueueMessage(models.Model):# Backstop no writer can bypass: state is a closed enum. Values sourced# from QueueMessageState (single source of truth; drift-tested).models.CheckConstraint(- check=models.Q(state__in=[_READY, _CLAIMED]),+ check=models.Q(state__in=[_READY, _CLAIMED, _SCHEDULED]),name="pg_queue_message_state_valid",),]@@ -114,6 +133,19 @@ class PgQueueMessage(models.Model):condition=models.Q(state=_CLAIMED),name="pg_queue_message_claimed_idx",),+ # PROMOTION path (UN-3843) — partial index over ONLY deferred rows. The+ # reaper's promotion sweep (`WHERE state='scheduled' AND+ # available_at<=now()`) walks this in due order, so a large backlog of+ # far-future rows costs one index seek, not a scan. Bounded by pending+ # delayed messages, which is a small set by construction (a delay is a+ # stagger or a retry backoff, not a queue). Keyed on `available_at` alone:+ # the sweep is time-ordered and queue-agnostic — it promotes every due row+ # in one statement rather than per queue.+ models.Index(+ F("available_at"),+ condition=models.Q(state=_SCHEDULED),+ name="pg_queue_message_scheduled_idx",+ ),]@@ -350,6 +382,84 @@ class PgPeriodicSchedule(models.Model):]+class PgPeriodicTask(models.Model):+ """Generic (non-pipeline) periodic, mirrored from ``django_celery_beat`` (UN-3796).++ The Beat-replacement sibling of :class:`PgPeriodicSchedule`. That table fires+ **pipeline** triggers: it is keyed on ``pipeline_id`` and its dispatcher rebuilds+ the fixed ``execute_pipeline_task`` argument list onto one fixed queue. Every+ other Beat periodic — ``dashboard_metrics.*``, log-history, audit, anything an+ operator adds — is an arbitrary ``(task, queue, args, kwargs)`` on a cron, which+ that shape cannot express.++ A **sibling table rather than a widened one**: ``PgPeriodicSchedule`` is live,+ flag-gated and mid-ramp, and widening it would mean a surrogate PK plus rewriting+ every existing row to carry columns only the other kind of schedule uses. The two+ genuinely differ in payload and target queue; they share only the cron mechanics,+ which live in the dispatcher, not the row.++ Keyed on ``name`` — ``PeriodicTask.name``, the natural mirror key and already+ unique in Beat, so reconciliation is a plain upsert with no id mapping to keep.++ Ownership and the no-double-fire guarantee are identical to the sibling's and+ depend on the same discipline: ``pg_owned`` defaults False, so the table is inert+ until a row is explicitly handed over, and handing one over MUST disable the+ matching Beat ``PeriodicTask`` in the same breath. A row flipped ``pg_owned=True``+ while Beat still has it enabled fires twice.++ Managed=True / generated migration, extension-free.+ """++ # = PeriodicTask.name. Beat's own uniqueness key, so the mirror needs no+ # surrogate id and reconciliation is an upsert on the same identity Beat uses.+ name = models.TextField(primary_key=True)+ # Dotted task path (PeriodicTask.task), e.g. "dashboard_metrics.aggregate_from_sources".+ task_name = models.TextField()+ # Target queue (PeriodicTask.queue). Unlike the pipeline scheduler's single+ # hardcoded queue, each periodic carries its own — dashboard_metrics.* go to+ # dashboard_metric_events, log-history to celery_periodic_logs, and so on.+ queue = models.TextField()+ # PeriodicTask.args / .kwargs, already decoded from Beat's JSON strings. Stored+ # decoded so the dispatcher builds a TaskPayload without re-parsing per tick, and+ # so a malformed value fails loudly at reconcile time rather than at fire time.+ task_args = models.JSONField(default=list)+ task_kwargs = models.JSONField(default=dict)+ # Cron for crontab-backed periodics. Beat also supports IntervalSchedule+ # (every N seconds/minutes); those are mirrored as an equivalent cron where one+ # exists and skipped with a warning where it does not — sub-minute intervals have+ # no cron expression, and silently coarsening one to a minute would change+ # behaviour. See reconcile_pg_schedules.+ cron_string = models.TextField()+ # Owning org, "" = none. Global periodics (dashboard_metrics.*) have no org, so+ # this is empty for every row today and the dispatcher writes it straight through+ # to the queue message's org_id. It exists now rather than later for one reason:+ # PgPeriodicSchedule carries organization_id, so without this column a future+ # unification of the two tables would be a schema redesign instead of a data+ # migration. Same no-NULL-text convention as elsewhere.+ org_id = models.TextField(blank=True, default="")+ # Mirrors PeriodicTask.enabled.+ enabled = models.BooleanField(default=True)+ # Per-row rollout switch; see the class docstring. Defaults False → inert.+ pg_owned = models.BooleanField(default=False)+ # Owned by the scheduler tick. NULL next_run_at = "record a baseline next time,+ # don't fire this cycle" — no burst when a row is handed over.+ last_run_at = models.DateTimeField(null=True, blank=True)+ next_run_at = models.DateTimeField(null=True, blank=True)+ created_at = models.DateTimeField(default=timezone.now)+ updated_at = models.DateTimeField(auto_now=True)++ class Meta:+ db_table = "pg_periodic_task"+ indexes = [+ # Drives the due scan, mirroring pg_periodic_schedule_due_idx:+ # WHERE pg_owned AND enabled AND (next_run_at IS NULL OR <= now()).+ models.Index(+ fields=["pg_owned", "enabled", "next_run_at"],+ name="pg_periodic_task_due_idx",+ ),+ ]++class PgTaskResult(models.Model):"""Request-reply result store for the executor RPC on PG.diff --git a/backend/pg_queue/producer.py b/backend/pg_queue/producer.pyindex 028fddfb6..3fc69bd19 100644--- a/backend/pg_queue/producer.py+++ b/backend/pg_queue/producer.py@@ -18,8 +18,11 @@ from __future__ import annotationsimport jsonimport logging+from datetime import UTC, datetime, timedeltafrom typing import Any+from django.utils import timezone+from pg_queue.models import PgQueueMessagefrom unstract.core.data_models import (FAIRNESS_DEFAULT_PRIORITY,@@ -27,9 +30,13 @@ from unstract.core.data_models import (FAIRNESS_MIN_PRIORITY,ContinuationSpec,FairnessPayload,+ QueueMessageState,TaskPayload,)+_READY = QueueMessageState.READY.value+_SCHEDULED = QueueMessageState.SCHEDULED.value+logger = logging.getLogger(__name__)# Fairness L3 priority default, re-exported under the producer's name so callers@@ -61,6 +68,33 @@ def _json_safe(value: Any) -> Any:return json.loads(json.dumps(value, default=str, allow_nan=False))+def _resolve_visibility(+ countdown: float | None, eta: datetime | None+) -> tuple[datetime, str]:+ """Map Celery-style ``countdown``/``eta`` onto ``(available_at, state)``.++ Returns ``(now, 'ready')`` for the immediate case — which is every pre-existing+ call site, so their rows stay byte-identical to before this parameter existed.+ A delay in the past or a non-positive countdown also resolves to immediate:+ "deliver no earlier than T" is already satisfied when T has passed, and a+ computed-zero stagger step must not cost a reaper tick.++ The caller has already rejected countdown+eta together.+ """+ now = timezone.now()+ if countdown is not None:+ target = now + timedelta(seconds=countdown)+ elif eta is not None:+ # A naive datetime would compare-fail against the aware `now` under+ # USE_TZ; treat it as UTC rather than raising on an otherwise valid call.+ target = eta if timezone.is_aware(eta) else timezone.make_aware(eta, UTC)+ else:+ return now, _READY+ if target <= now:+ return now, _READY+ return target, _SCHEDULED++def enqueue_task(*,task_name: str,@@ -74,6 +108,8 @@ def enqueue_task(on_success: ContinuationSpec | None = None,on_error: ContinuationSpec | None = None,task_id: str | None = None,+ countdown: float | None = None,+ eta: datetime | None = None,) -> int:"""Enqueue a task onto the PG queue; returns the new ``msg_id``.@@ -92,7 +128,25 @@ def enqueue_task(``on_error`` as the failed id (Celery ``link_error`` parity). Mutuallyexclusive with ``reply_key`` — passing both is rejected (the consumer checks``reply_key`` first and would silently drop the callback).++ ``countdown`` (seconds from now) / ``eta`` (absolute time) defer delivery,+ mirroring Celery's kwargs of the same names; they are mutually exclusive. A+ deferred row is written ``state='scheduled'`` and is invisible to the claim+ until the reaper promotes it once ``available_at`` passes — so the guarantee is+ **not before** the requested time, at reaper-tick granularity (default 5s), not+ exactly at it. That matches Celery, whose countdown is also approximate, and is+ the right semantic for a stagger or a retry backoff: never early. A+ non-positive ``countdown`` or a past ``eta`` is treated as "send now" rather+ than an error, so a computed-zero stagger step (``i * delay`` with ``i == 0``)+ stays on the immediate path instead of paying a reaper tick.++ **Requires a running reaper.** Nothing else promotes a scheduled row. The reaper+ is already the mandatory singleton for crash redelivery, so this adds no new+ deployment dependency — but a queue that uses delays inherits the reaper's+ liveness alert (PG_QUEUE_CLAIM_STATE_RUNBOOK)."""+ if countdown is not None and eta is not None:+ raise ValueError("countdown and eta are mutually exclusive")if reply_key is not None and (on_success is not None or on_error is not None):raise ValueError("reply_key (request-reply) and on_success/on_error (callback) are "@@ -104,6 +158,7 @@ def enqueue_task(f"[{FAIRNESS_MIN_PRIORITY}, {FAIRNESS_MAX_PRIORITY}]: {priority!r}")pg_queue = queue or DEFAULT_GENERAL_QUEUE+ available_at, state = _resolve_visibility(countdown, eta)# Mirror the worker _enqueue_pg path: log the failure with breadcrumbs before# it propagates, so a DB/constraint/serialization error isn't mislabeled by# the caller's broad handler. Message construction (the _json_safe coercions)@@ -140,6 +195,8 @@ def enqueue_task(message=message,org_id=org_id or "",priority=priority,+ available_at=available_at,+ state=state,)except Exception:logger.exception(diff --git a/backend/pg_queue/tests/test_converge_pg_scheduler.py b/backend/pg_queue/tests/test_converge_pg_scheduler.pynew file mode 100644index 000000000..f0edae1fe--- /dev/null+++ b/backend/pg_queue/tests/test_converge_pg_scheduler.py@@ -0,0 +1,91 @@+"""The converge command routes by PG_SCHEDULER_ENABLED — in BOTH directions.++Thin glue, but the branch is load-bearing: it is what makes a rollback a values+change rather than a procedure someone has to remember under pressure. Getting the+direction wrong would either adopt on a rollback (the opposite of intent) or leave+the reverse a no-op — the exact failure that stranded a pipeline on integration and+made Beat and the PG scheduler both fire it.++The sub-commands are stubbed: their own behaviour is covered by+test_reconcile_pg_schedules_command / test_mirror_pg_periodic_tasks. What is pinned+here is *which* is called, with *which* flags.+"""++from unittest.mock import patch++import pytest+from django.core.management import call_command++_CMD = "pg_queue.management.commands.converge_pg_scheduler"+++def _run(monkeypatch, *, gate: bool, periodics: bool = False, dry_run: bool = False):+ """Run the command with the gate set, returning the sub-command calls made."""+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true" if gate else "false")+ with patch(f"{_CMD}.call_command") as sub:+ args = ["converge_pg_scheduler"]+ if periodics:+ args.append("--periodics")+ if dry_run:+ args.append("--dry-run")+ call_command(*args)+ # {command_name: kwargs} — no sub-command is invoked twice in either direction.+ return {c.args[0]: c.kwargs for c in sub.call_args_list}+++class TestDirection:+ def test_gate_on_adopts_pipelines(self, monkeypatch):+ calls = _run(monkeypatch, gate=True)+ assert "reconcile_pg_schedules" in calls+ # NOT mirror-only: that is the inert mode, which would adopt nothing.+ assert not calls["reconcile_pg_schedules"].get("mirror_only")+ assert not calls["reconcile_pg_schedules"].get("release_stale")++ def test_gate_off_releases_pipelines_back_to_beat(self, monkeypatch):+ calls = _run(monkeypatch, gate=False)+ kwargs = calls["reconcile_pg_schedules"]+ # Both halves: mirror stays inert-safe, release_stale is the actual rollback.+ assert kwargs["mirror_only"] is True+ assert kwargs["release_stale"] is True++ def test_an_unset_gate_converges_to_beat(self, monkeypatch):+ """Absent must mean Beat, not "do nothing" — the default has to be the+ safe direction, since that is what an environment that never opted in has.+ """+ monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ with patch(f"{_CMD}.call_command") as sub:+ call_command("converge_pg_scheduler")+ kwargs = {c.args[0]: c.kwargs for c in sub.call_args_list}+ assert kwargs["reconcile_pg_schedules"]["release_stale"] is True+++class TestPeriodicsAreOptIn:+ def test_periodics_are_untouched_by_default(self, monkeypatch):+ """Deferring metrics must be real.++ Adopting them requires workerPgMetrics to be deployed; doing it implicitly+ would fire dashboard_metrics.* into a queue with no consumer.+ """+ assert "mirror_pg_periodic_tasks" not in _run(monkeypatch, gate=True)+ assert "mirror_pg_periodic_tasks" not in _run(monkeypatch, gate=False)++ def test_gate_on_with_periodics_adopts_every_mirrored_row(self, monkeypatch):+ calls = _run(monkeypatch, gate=True, periodics=True)+ # Empty list = "all rows" (nargs="*"); None would mean the flag was absent.+ assert calls["mirror_pg_periodic_tasks"]["adopt"] == []++ def test_gate_off_with_periodics_releases_them(self, monkeypatch):+ calls = _run(monkeypatch, gate=False, periodics=True)+ assert calls["mirror_pg_periodic_tasks"]["release"] == []+ assert "adopt" not in calls["mirror_pg_periodic_tasks"]+++class TestDryRunReaches_Everything:+ """A dry run that silently writes through one sub-command is worse than none."""++ @pytest.mark.parametrize("gate", [True, False])+ def test_dry_run_propagates_to_every_sub_command(self, monkeypatch, gate):+ calls = _run(monkeypatch, gate=gate, periodics=True, dry_run=True)+ assert calls, "no sub-commands were invoked"+ for name, kwargs in calls.items():+ assert kwargs.get("dry_run") is True, namediff --git a/backend/pg_queue/tests/test_mirror_pg_periodic_tasks.py b/backend/pg_queue/tests/test_mirror_pg_periodic_tasks.pynew file mode 100644index 000000000..d5fe34290--- /dev/null+++ b/backend/pg_queue/tests/test_mirror_pg_periodic_tasks.py@@ -0,0 +1,371 @@+"""Unit tests for the non-pipeline Beat mirror (UN-3796).++DB-free by construction: every decision lives in ``plan_mirror`` /+``cron_from_periodic_task``, which take anything shaped like a ``PeriodicTask``.+That matters here — no backend DB-bound test currently runs in this repo (the+multi-tenant schema isn't created for the test database, and pytest is not wired+into CI for backend at all), so a test written against the ORM would look like+coverage while never executing.++What's worth pinning is the exclusions (what must NOT be mirrored, and why) and the+cron conversion (where a wrong answer silently changes how often a job runs).+"""++import json+from types import SimpleNamespace+from unittest.mock import MagicMock, patch++import pytest+from django.core.management import call_command++from pg_queue.management.commands.mirror_pg_periodic_tasks import (+ cron_from_periodic_task,+ plan_mirror,+)++# django_celery_beat's IntervalSchedule period values are plain strings.+SECONDS, MINUTES, HOURS, DAYS = "seconds", "minutes", "hours", "days"+++def _crontab(minute="0", hour="2", dom="*", moy="*", dow="*"):+ return SimpleNamespace(+ minute=minute, hour=hour, day_of_month=dom, month_of_year=moy, day_of_week=dow+ )+++# Sentinel so an explicit `crontab=None` (a solar/clocked periodic) is+# distinguishable from "caller didn't say", which defaults to a plain crontab.+_UNSET = object()+++def _task(+ name="t",+ task="some.task",+ queue="q",+ args="[]",+ kwargs="{}",+ enabled=True,+ crontab=_UNSET,+ interval=None,+):+ if crontab is _UNSET:+ crontab = None if interval else _crontab()+ return SimpleNamespace(+ name=name,+ task=task,+ queue=queue,+ args=args,+ kwargs=kwargs,+ enabled=enabled,+ crontab=crontab,+ interval=interval,+ )+++def _interval(every, period):+ return SimpleNamespace(every=every, period=period)+++class TestCronConversion:+ def test_crontab_is_reconstructed_field_for_field(self):+ assert cron_from_periodic_task(_task(crontab=_crontab("30", "4"))) == "30 4 * * *"++ @pytest.mark.parametrize(+ "every,period,expected",+ [+ (15, MINUTES, "*/15 * * * *"), # dashboard_metrics.aggregate_from_sources+ (1, MINUTES, "*/1 * * * *"),+ (30, MINUTES, "*/30 * * * *"),+ (2, HOURS, "0 */2 * * *"),+ (12, HOURS, "0 */12 * * *"),+ # every==1 day is the ONLY expressible day interval; emitted as a plain+ # daily rather than `*/1` so the stored cron says what it means.+ (1, DAYS, "0 0 * * *"),+ ],+ )+ def test_interval_maps_to_an_exact_step_cron(self, every, period, expected):+ assert cron_from_periodic_task(_task(interval=_interval(every, period))) == expected++ def test_second_resolution_interval_is_refused_not_approximated(self):+ # The important negative: cron has no sub-minute field. Rounding a+ # 30-second periodic up to a minute would silently halve how often it runs,+ # so it must be refused. This is the real `workflow_log_history_v2` shape.+ assert cron_from_periodic_task(_task(interval=_interval(30, SECONDS))) == ""++ @pytest.mark.parametrize(+ "every,period", [(90, MINUTES), (36, HOURS), (40, DAYS), (0, MINUTES)]+ )+ def test_step_too_large_for_one_field_is_refused(self, every, period):+ # `*/90` in a 0-59 minute field does not mean "every 90 minutes".+ assert cron_from_periodic_task(_task(interval=_interval(every, period))) == ""++ @pytest.mark.parametrize(+ "every,period,fires_instead",+ [+ (7, MINUTES, ":00 :07 … :56 then :00 — a 4-minute gap, not 7"),+ (45, MINUTES, ":00 :45 then :00 — roughly twice as often"),+ (8, MINUTES, ":00 :08 … :56 then :00 — a 4-minute gap"),+ (25, MINUTES, ":00 :25 :50 then :00 — a 10-minute gap"),+ (5, HOURS, "0 5 10 15 20 then 0 — a 4-hour gap, not 5"),+ (7, HOURS, "0 7 14 21 then 0 — a 3-hour gap"),+ (9, HOURS, "0 9 18 then 0 — a 6-hour gap"),+ ],+ )+ def test_an_interval_that_does_not_divide_its_field_is_refused(+ self, every, period, fires_instead+ ):+ """`*/N` restarts at each field boundary, so it is only "every N" when N+ divides the range. The old code range-checked (`every < 60` / `< 24`) while+ the docstring claimed exactness, so these mirrored to a cron that fires at+ the wrong rate — silently, and only at the boundary.++ Refusing sends them down plan_mirror's skip-and-explain path, which is what+ already happens for second-resolution intervals: they stay on Beat, visibly,+ instead of being adopted at a frequency nobody chose.+ """+ assert cron_from_periodic_task(_task(interval=_interval(every, period))) == "", (+ f"every {every} {period} must be refused — `*/{every}` fires {fires_instead}"+ )++ @pytest.mark.parametrize("every", [2, 3, 7, 15, 31])+ def test_multi_day_intervals_are_refused_because_months_vary(self, every):+ """`0 0 */N * *` restarts every month and months are 28-31 days, so the+ boundary gap depends on the month and the year. `0 0 */7 * *` fires on the+ 1st, 8th, 15th, 22nd, 29th, then the 1st again — 2 to 4 days later. There is+ no correct cron for "every N days" at N > 1, so only every==1 is expressible.+ """+ assert cron_from_periodic_task(_task(interval=_interval(every, DAYS))) == ""++ def test_no_schedule_at_all_is_refused(self):+ # solar/clocked periodics have neither crontab nor interval.+ assert cron_from_periodic_task(_task(crontab=None, interval=None)) == ""+++class TestPlanMirror:+ def test_mirrors_with_its_own_queue_and_decoded_kwargs(self):+ plan = plan_mirror(+ _task(+ name="dashboard_metrics_cleanup_hourly",+ task="dashboard_metrics.cleanup_hourly_data",+ queue="dashboard_metric_events",+ kwargs=json.dumps({"retention_days": 30}),+ )+ )+ assert plan.should_mirror+ # Each periodic carries its OWN target queue — the whole reason this can't+ # reuse the pipeline scheduler, which enqueues onto one fixed queue.+ assert plan.fields["queue"] == "dashboard_metric_events"+ assert plan.fields["task_kwargs"] == {"retention_days": 30}+ assert plan.fields["cron_string"] == "0 2 * * *"++ @pytest.mark.parametrize(+ "task_path",+ [+ "scheduler.tasks.execute_pipeline_task",+ # A real Beat table was found carrying this legacy path; excluding by+ # name convention alone would have let it through into a second owner.+ "execute_pipeline_task_v2",+ # Celery's own result-backend housekeeping — nothing registers it on PG,+ # and what it cleans stops existing once Celery is off.+ "celery.backend_cleanup",+ ],+ )+ def test_excluded_task_paths_are_never_mirrored(self, task_path):+ plan = plan_mirror(_task(task=task_path))+ assert not plan.should_mirror+ assert "owned elsewhere" in plan.skip_reason++ def test_uncronnable_schedule_is_skipped_with_a_reason(self):+ plan = plan_mirror(_task(interval=_interval(30, SECONDS)))+ assert not plan.should_mirror+ assert "no cron equivalent" in plan.skip_reason++ def test_malformed_args_is_skipped_rather_than_mirrored_empty(self):+ # Beat stores args as JSON text, but a hand-edited row can hold a Python+ # repr (a real one was found). Defaulting to [] would fire the task with the+ # wrong arguments — silently, forever.+ plan = plan_mirror(_task(args="['not', 'json']"))+ assert not plan.should_mirror+ assert "malformed" in plan.skip_reason++ def test_missing_queue_falls_back_the_way_beat_does(self):+ assert plan_mirror(_task(queue=None)).fields["queue"] == "celery"++ def test_empty_args_and_kwargs_decode_to_empty_containers(self):+ fields = plan_mirror(_task(args="", kwargs=None)).fields+ assert fields["task_args"] == [] and fields["task_kwargs"] == {}++ def test_disabled_state_is_carried_across(self):+ # A row disabled in Beat must mirror as disabled, or adopting it would start+ # running something an operator had deliberately turned off.+ assert plan_mirror(_task(enabled=False)).fields["enabled"] is False+++class TestBeatReloadSignal:+ """A hand-over must tell Beat to reload, or the atomicity is worthless.++ `PeriodicTask.objects.filter(...).update(...)` is a bulk update: it bypasses+ django-celery-beat's post_save signal, so `PeriodicTasks.last_update` never bumps+ and `DatabaseScheduler` keeps running from its stale in-memory copy. The DB would+ say "disabled" while Beat carried on firing — a double fire alongside the PG+ scheduler, which is the exact failure doing both halves in one transaction exists+ to prevent. `--release` fails the mirror way: Beat never resumes.++ Pinned here because the symptom is invisible in the DB — you only see it in+ duplicated side effects.+ """++ _CMD = "pg_queue.management.commands.mirror_pg_periodic_tasks"++ def _run(self, flag):+ # `enabled` mirrors Beat's value and is what --release restores; a real+ # PgPeriodicTask always has it, so the stand-in must too.+ row = SimpleNamespace(+ name="h", pg_owned=(flag == "--release"), next_run_at=None, enabled=True+ )+ with (+ patch(f"{self._CMD}.PgPeriodicTask") as Model,+ patch(f"{self._CMD}.PeriodicTask") as Beat,+ patch(f"{self._CMD}.PeriodicTasks") as BeatSignal,+ patch(f"{self._CMD}.transaction.atomic"),+ ):+ qs = MagicMock()+ qs.iterator.return_value = [row]+ qs.values_list.return_value = ["h"]+ qs.filter.return_value = qs+ Model.objects.all.return_value.order_by.return_value = qs+ # No Beat rows to mirror; we only exercise the ownership flip.+ Beat.objects.exclude.return_value.select_related.return_value.order_by.return_value.iterator.return_value = []+ row.save = MagicMock()+ call_command("mirror_pg_periodic_tasks", flag)+ return Beat, BeatSignal++ @pytest.mark.parametrize("flag", ["--adopt", "--release"])+ def test_ownership_flip_bumps_the_beat_reload_signal(self, flag):+ Beat, BeatSignal = self._run(flag)+ Beat.objects.filter.return_value.update.assert_called_once()+ BeatSignal.update_changed.assert_called_once()+++class TestReleaseRestoresRatherThanResurrects:+ """A rollback restores the previous state; it must not invent a new one.++ `--release` wrote `enabled=True` unconditionally, so a periodic an operator had+ deliberately switched OFF in Beat before the migration came back **on** after a+ rollback — silently re-arming a job someone had stopped on purpose. The+ pre-migration value is already recorded: `plan_mirror` copies `task.enabled` into+ the mirror row, so `row.enabled` is Beat's own value.+ """++ _CMD = "pg_queue.management.commands.mirror_pg_periodic_tasks"++ def _flip(self, flag: str, mirrored_enabled: bool = True):+ row = SimpleNamespace(+ name="h",+ pg_owned=(flag == "--release"),+ next_run_at=None,+ enabled=mirrored_enabled,+ )+ with (+ patch(f"{self._CMD}.PgPeriodicTask") as Model,+ patch(f"{self._CMD}.PeriodicTask") as Beat,+ patch(f"{self._CMD}.PeriodicTasks"),+ patch(f"{self._CMD}.transaction.atomic"),+ ):+ qs = MagicMock()+ qs.iterator.return_value = [row]+ qs.values_list.return_value = ["h"]+ qs.filter.return_value = qs+ Model.objects.all.return_value.order_by.return_value = qs+ Beat.objects.exclude.return_value.select_related.return_value.order_by.return_value.iterator.return_value = []+ row.save = MagicMock()+ call_command("mirror_pg_periodic_tasks", flag)+ return Beat.objects.filter.return_value.update.call_args.kwargs++ def _release(self, mirrored_enabled: bool):+ return self._flip("--release", mirrored_enabled)++ def test_a_row_that_was_enabled_is_restored_enabled(self):+ assert self._release(mirrored_enabled=True)["enabled"] is True++ def test_a_row_that_was_DISABLED_stays_disabled(self):+ # The regression: blanket `enabled=True` re-armed a job someone stopped.+ assert self._release(mirrored_enabled=False)["enabled"] is False+++class TestReleaseBaselinesBeatsClock:+ """Release must reset Beat's clock, or Beat replays every missed interval.++ DatabaseScheduler keeps no next_run_at — it derives due-ness from+ ``PeriodicTask.last_run_at`` against the crontab. A periodic that spent days+ PG-owned still carries the last_run_at from before the hand-over, so the moment+ ``enabled`` flips back it is overdue by every interval it missed and Beat fires+ them all at once.++ Observed on integration 2026-08-24: releasing the fleet fired all three+ ``dashboard_metrics.*`` plus four pipelines within 30 ms of "Released to Beat".+ This is the exact mirror of the next_run_at baseline on the adopt side+ (``scheduler/ownership.py``, OSS 2088d6962) — that half was fixed, this one was+ not, and a comment there wrongly claimed Beat does not catch up.++ These assertions previously read ``== {"enabled": ...}`` on the whole kwargs dict.+ Changed deliberately, not loosened: the write now carries a second key, and the+ per-key assertions below plus the two new cases pin strictly more than the exact+ dict did.+ """++ _CMD = "pg_queue.management.commands.mirror_pg_periodic_tasks"++ def _flip(self, flag: str):+ return TestReleaseRestoresRatherThanResurrects()._flip(flag)++ def test_release_stamps_last_run_at(self):+ kwargs = self._flip("--release")+ assert "last_run_at" in kwargs, (+ "release must baseline Beat's clock; without it DatabaseScheduler sees "+ "every missed interval as overdue and replays them"+ )+ assert kwargs["last_run_at"] is not None++ def test_adopt_does_NOT_touch_last_run_at(self):+ """On adopt Beat is being switched OFF, so its clock is irrelevant — and+ overwriting it would destroy the value a later release needs to restore."""+ assert "last_run_at" not in self._flip("--adopt")+++class TestMirrorDoesNotClobberAnAdoptedRow:+ """Re-mirroring after adoption must not strand the row with no firer.++ `enabled` is the one mirrored field that stops tracking Beat once PG owns the row:+ after `--adopt`, Beat's copy is False *by definition*. Copying that back leaves+ `pg_owned=True, enabled=False`, which matches NEITHER firer — the PG tick selects+ `WHERE pg_owned AND enabled`, and Beat's row is disabled. The periodic stops+ silently, and the value needed to release it correctly is gone.+ """++ _CMD = "pg_queue.management.commands.mirror_pg_periodic_tasks"++ def _mirror_with(self, already_adopted: bool):+ beat_row = _task(name="h", enabled=False, crontab=_crontab())+ with (+ patch(f"{self._CMD}.PgPeriodicTask") as Model,+ patch(f"{self._CMD}.PeriodicTask") as Beat,+ patch(f"{self._CMD}.PeriodicTasks"),+ ):+ Beat.objects.exclude.return_value.select_related.return_value.order_by.return_value.iterator.return_value = [+ beat_row+ ]+ Model.objects.filter.return_value.exists.return_value = already_adopted+ call_command("mirror_pg_periodic_tasks")+ return Model.objects.update_or_create.call_args.kwargs["defaults"]++ def test_an_adopted_row_keeps_its_own_enabled(self):+ assert "enabled" not in self._mirror_with(already_adopted=True)++ def test_a_beat_owned_row_still_tracks_beat(self):+ # The normal path must keep mirroring enabled, or a pause in Beat is missed.+ assert self._mirror_with(already_adopted=False)["enabled"] is False++ def test_cron_still_tracks_beat_even_when_adopted(self):+ # Only `enabled` diverges — a schedule edit made while PG owns it must land.+ assert "cron_string" in self._mirror_with(already_adopted=True)diff --git a/backend/pg_queue/tests/test_producer.py b/backend/pg_queue/tests/test_producer.pyindex c94d219f0..246855a95 100644--- a/backend/pg_queue/tests/test_producer.py+++ b/backend/pg_queue/tests/test_producer.py@@ -10,8 +10,10 @@ import uuidfrom unittest.mock import MagicMock, patchimport pytest+from django.utils import timezone as django_timezonefrom pg_queue import producer+from unstract.core.data_models import QueueMessageState_MODEL = "pg_queue.producer.PgQueueMessage"@@ -176,3 +178,90 @@ class TestEnqueueTask:)msg = model.objects.create.call_args.kwargs["message"]assert msg["on_success"]["kwargs"]["callback_kwargs"]["doc_id"] == str(uid)+++class TestDelayedVisibility:+ """UN-3843 — ``countdown``/``eta`` defer delivery via ``scheduled`` + ``available_at``.++ A deferred row is written ``state='scheduled'`` so it is absent from the claim's+ partial index; the reaper promotes it when due. These pin the producer half of+ that contract — that the right ``(available_at, state)`` pair reaches the row, and+ that every non-deferred call still writes exactly what it wrote before this+ parameter existed.+ """++ @staticmethod+ def _create_kwargs(model):+ return model.objects.create.call_args.kwargs++ def test_no_delay_is_unchanged_and_ready(self):+ # The zero-regression case: every pre-existing call site lands here.+ before = django_timezone.now()+ with patch(_MODEL) as model:+ model.objects.create.return_value = MagicMock(msg_id=1)+ producer.enqueue_task(task_name="t", queue="q")+ kw = self._create_kwargs(model)+ assert kw["state"] == QueueMessageState.READY.value+ assert before <= kw["available_at"] <= django_timezone.now()++ def test_countdown_defers_and_marks_scheduled(self):+ with patch(_MODEL) as model:+ model.objects.create.return_value = MagicMock(msg_id=1)+ before = django_timezone.now()+ producer.enqueue_task(task_name="t", queue="q", countdown=90)+ kw = self._create_kwargs(model)+ assert kw["state"] == QueueMessageState.SCHEDULED.value+ # ~90s out; generous window so a slow CI box can't flake it.+ delta = (kw["available_at"] - before).total_seconds()+ assert 89 <= delta <= 95++ def test_eta_defers_and_marks_scheduled(self):+ eta = django_timezone.now() + datetime.timedelta(minutes=5)+ with patch(_MODEL) as model:+ model.objects.create.return_value = MagicMock(msg_id=1)+ producer.enqueue_task(task_name="t", queue="q", eta=eta)+ kw = self._create_kwargs(model)+ assert kw["state"] == QueueMessageState.SCHEDULED.value+ assert kw["available_at"] == eta++ def test_naive_eta_is_read_as_utc(self):+ # USE_TZ makes `now()` aware; a naive eta would raise on comparison. Treat+ # it as UTC rather than rejecting an otherwise valid call.+ naive = (+ django_timezone.now() + datetime.timedelta(hours=1)+ ).replace(tzinfo=None)+ with patch(_MODEL) as model:+ model.objects.create.return_value = MagicMock(msg_id=1)+ producer.enqueue_task(task_name="t", queue="q", eta=naive)+ kw = self._create_kwargs(model)+ assert kw["state"] == QueueMessageState.SCHEDULED.value+ assert kw["available_at"].tzinfo is not None++ @pytest.mark.parametrize("countdown", [0, -5])+ def test_non_positive_countdown_stays_on_the_immediate_path(self, countdown):+ # A stagger computes `i * delay`; step 0 must not pay a reaper tick just to+ # become claimable.+ with patch(_MODEL) as model:+ model.objects.create.return_value = MagicMock(msg_id=1)+ producer.enqueue_task(task_name="t", queue="q", countdown=countdown)+ assert self._create_kwargs(model)["state"] == QueueMessageState.READY.value++ def test_past_eta_stays_on_the_immediate_path(self):+ past = django_timezone.now() - datetime.timedelta(minutes=1)+ with patch(_MODEL) as model:+ model.objects.create.return_value = MagicMock(msg_id=1)+ producer.enqueue_task(task_name="t", queue="q", eta=past)+ assert self._create_kwargs(model)["state"] == QueueMessageState.READY.value++ def test_countdown_and_eta_together_are_rejected(self):+ with patch(_MODEL) as model:+ model.objects.create.return_value = MagicMock(msg_id=1)+ with pytest.raises(ValueError, match="mutually exclusive"):+ producer.enqueue_task(+ task_name="t",+ queue="q",+ countdown=10,+ eta=django_timezone.now(),+ )+ # Rejected before any row is written — no half-enqueued message.+ model.objects.create.assert_not_called()diff --git a/backend/pg_queue/tests/test_reconcile_pg_schedules_command.py b/backend/pg_queue/tests/test_reconcile_pg_schedules_command.pyindex 081aff2cf..04d8d0775 100644--- a/backend/pg_queue/tests/test_reconcile_pg_schedules_command.py+++ b/backend/pg_queue/tests/test_reconcile_pg_schedules_command.py@@ -11,6 +11,10 @@ import pytestfrom django.core.management import call_commandfrom django.core.management.base import CommandError+from pg_queue.management.commands.reconcile_pg_schedules import (+ DEFAULT_BATCH_SIZE,+)+_CMD = "pg_queue.management.commands.reconcile_pg_schedules"@@ -35,6 +39,32 @@ def _row(pid, org="org", enabled=True):return m+def _periodic_tasks(rows):+ """Mock the PeriodicTask queryset chain the command now uses.++ The command reads `.filter(...).select_related(...).order_by(...).iterator(...)`+ so the scan is chunked; these tests pin behaviour, not the chain, so the helper+ absorbs the plumbing.+ """+ qs = MagicMock()+ qs.select_related.return_value.order_by.return_value.iterator.return_value = rows+ return qs+++def _sched_ids(ids):+ """Mock `.values_list("pipeline_id", flat=True).iterator(...)`."""+ vl = MagicMock()+ vl.iterator.return_value = ids+ return vl+++def _sched_rows(rows):+ """Mock `.order_by("pk").iterator(...)`."""+ qs = MagicMock()+ qs.iterator.return_value = rows+ return qs++class TestReconcileCommand:def test_backfills_only_unmirrored_and_reconciles(self):pt_new = _pt("pid-new", args='["wf", "org", "", "", "pid-new", false, "n"]')@@ -45,16 +75,35 @@ class TestReconcileCommand:patch(f"{_CMD}.mirror_periodic_schedule_upsert") as upsert,patch(f"{_CMD}.reconcile_ownership_for", return_value=False) as reconcile,):- PT.objects.filter.return_value = [pt_new, pt_exists]+ PT.objects.filter.return_value = _periodic_tasks([pt_new, pt_exists])# pid-exists already mirrored; pid-new not (one prefetch query).- Sched.objects.values_list.return_value = ["pid-exists"]- Sched.objects.all.return_value = [_row("pid-new"), _row("pid-exists")]+ Sched.objects.values_list.return_value = _sched_ids(["pid-exists"])+ Sched.objects.order_by.return_value = _sched_rows([_row("pid-new"), _row("pid-exists")])call_command("reconcile_pg_schedules")upsert.assert_called_once() # only the unmirrored one backfilledassert upsert.call_args.kwargs["pipeline_id"] == "pid-new"assert reconcile.call_count == 2 # both rows reconciled+ def test_mirror_only_backfills_but_never_reconciles(self):+ """--mirror-only is what deploy automation runs, so it must be safe at ANY+ flag state. The reconcile it skips is the step that disables Beat rows once+ the rollout is on — an unattended job must never make that change."""+ pt_new = _pt("pid-new", args='["wf", "org", "", "", "pid-new", false, "n"]')+ with (+ patch(f"{_CMD}.PeriodicTask") as PT,+ patch(f"{_CMD}.PgPeriodicSchedule") as Sched,+ patch(f"{_CMD}.mirror_periodic_schedule_upsert") as upsert,+ patch(f"{_CMD}.reconcile_ownership_for") as reconcile,+ ):+ PT.objects.filter.return_value = _periodic_tasks([pt_new])+ Sched.objects.values_list.return_value = _sched_ids([])+ Sched.objects.order_by.return_value = _sched_rows([_row("pid-new")])+ call_command("reconcile_pg_schedules", "--mirror-only")++ upsert.assert_called_once() # the backfill still happens+ reconcile.assert_not_called() # the ownership flip does not+def test_malformed_args_skipped_not_fatal(self):bad = _pt("pid-bad", args="{ this is not json")good = _pt("pid-good", args='["wf", "org", "", "", "pid-good", false, "n"]')@@ -64,9 +113,9 @@ class TestReconcileCommand:patch(f"{_CMD}.mirror_periodic_schedule_upsert") as upsert,patch(f"{_CMD}.reconcile_ownership_for", return_value=False),):- PT.objects.filter.return_value = [bad, good]- Sched.objects.values_list.return_value = []- Sched.objects.all.return_value = []+ PT.objects.filter.return_value = _periodic_tasks([bad, good])+ Sched.objects.values_list.return_value = _sched_ids([])+ Sched.objects.order_by.return_value = _sched_rows([])# Must not raise despite the bad row.call_command("reconcile_pg_schedules")@@ -82,9 +131,9 @@ class TestReconcileCommand:patch(f"{_CMD}.mirror_periodic_schedule_upsert") as upsert,patch(f"{_CMD}.reconcile_ownership_for", return_value=False),):- PT.objects.filter.return_value = [weird]- Sched.objects.values_list.return_value = []- Sched.objects.all.return_value = []+ PT.objects.filter.return_value = _periodic_tasks([weird])+ Sched.objects.values_list.return_value = _sched_ids([])+ Sched.objects.order_by.return_value = _sched_rows([])call_command("reconcile_pg_schedules")upsert.assert_not_called()@@ -97,11 +146,11 @@ class TestReconcileCommand:patch(f"{_CMD}.reconcile_ownership_for") as reconcile,patch(f"{_CMD}.resolve_schedule_owner", return_value=True) as resolve,):- PT.objects.filter.return_value = [- _pt("pid-1", args='["wf", "org", "", "", "pid-1", false, "n"]')- ]- Sched.objects.values_list.return_value = []- Sched.objects.all.return_value = [_row("pid-1")]+ PT.objects.filter.return_value = _periodic_tasks(+ [_pt("pid-1", args='["wf", "org", "", "", "pid-1", false, "n"]')]+ )+ Sched.objects.values_list.return_value = _sched_ids([])+ Sched.objects.order_by.return_value = _sched_rows([_row("pid-1")])call_command("reconcile_pg_schedules", "--dry-run")upsert.assert_not_called() # no backfill write@@ -115,8 +164,65 @@ class TestReconcileCommand:patch(f"{_CMD}.mirror_periodic_schedule_upsert"),patch(f"{_CMD}.reconcile_ownership_for", return_value=None), # failed):- PT.objects.filter.return_value = []- Sched.objects.values_list.return_value = []- Sched.objects.all.return_value = [_row("pid-1")]+ PT.objects.filter.return_value = _periodic_tasks([])+ Sched.objects.values_list.return_value = _sched_ids([])+ Sched.objects.order_by.return_value = _sched_rows([_row("pid-1")])with pytest.raises(CommandError):call_command("reconcile_pg_schedules")+++class TestChunking:+ """Both loops must stay bounded — there is one row per scheduled pipeline, so an+ unbounded scan holds the whole pipeline population in memory (twice, before this).+ Pinned so a later refactor can't quietly drop `.iterator()`.+ """++ def _run(self, extra_args=()):+ with (+ patch(f"{_CMD}.PeriodicTask") as PT,+ patch(f"{_CMD}.PgPeriodicSchedule") as Sched,+ patch(f"{_CMD}.mirror_periodic_schedule_upsert"),+ patch(f"{_CMD}.reconcile_ownership_for", return_value=False),+ ):+ PT.objects.filter.return_value = _periodic_tasks([])+ Sched.objects.values_list.return_value = _sched_ids([])+ Sched.objects.order_by.return_value = _sched_rows([])+ call_command("reconcile_pg_schedules", *extra_args)+ return PT, Sched++ def test_both_scans_are_chunked_with_the_default_batch_size(self):+ PT, Sched = self._run()+ pt_chain = PT.objects.filter.return_value.select_related.return_value+ pt_chain.order_by.return_value.iterator.assert_called_once_with(+ chunk_size=DEFAULT_BATCH_SIZE+ )+ Sched.objects.order_by.return_value.iterator.assert_called_once_with(+ chunk_size=DEFAULT_BATCH_SIZE+ )++ def test_mirrored_id_prefetch_streams_rather_than_materialising(self):+ # This one used to build a set from the full table with no bound.+ _, Sched = self._run()+ Sched.objects.values_list.assert_called_once_with("pipeline_id", flat=True)+ Sched.objects.values_list.return_value.iterator.assert_called_once_with(+ chunk_size=DEFAULT_BATCH_SIZE+ )++ def test_batch_size_flag_is_honoured(self):+ PT, Sched = self._run(("--batch-size", "7"))+ pt_chain = PT.objects.filter.return_value.select_related.return_value+ pt_chain.order_by.return_value.iterator.assert_called_once_with(chunk_size=7)+ Sched.objects.order_by.return_value.iterator.assert_called_once_with(chunk_size=7)++ def test_scans_are_ordered_so_a_chunked_walk_is_deterministic(self):+ # Without an ORDER BY, a chunked scan has no guaranteed row order between+ # batches, so rows can be visited twice or not at all.+ PT, Sched = self._run()+ pt_chain = PT.objects.filter.return_value.select_related.return_value+ assert pt_chain.order_by.call_args[0] == ("pk",)+ assert Sched.objects.order_by.call_args[0] == ("pk",)++ @pytest.mark.parametrize("bad", ["0", "-1"])+ def test_non_positive_batch_size_is_rejected(self, bad):+ with pytest.raises(CommandError, match="--batch-size"):+ call_command("reconcile_pg_schedules", "--batch-size", bad)diff --git a/backend/scheduler/ownership.py b/backend/scheduler/ownership.pyindex 9c1e1f287..c7c9c2817 100644--- a/backend/scheduler/ownership.py+++ b/backend/scheduler/ownership.py@@ -11,10 +11,17 @@ Doing both in one transaction is what makes "never double-fires" real (it was*conditional* on this slice): a ``pg_owned`` row always has its Beat``PeriodicTask`` disabled, so the two can't both fire.-Inert by default: ``resolve_schedule_owner`` fails closed to Beat-(``pg_owned=False``) until ops ramps the single ``pg_queue_enabled`` Flipt flag — so-reconciling on every schedule edit is a no-op (everything stays Beat-owned) until the-rollout starts.+**Two gates, and both must be on.** ``PG_SCHEDULER_ENABLED`` (env, default off) comes+first; the ``pg_queue_enabled`` Flipt flag then decides per schedule, failing closed to+Beat on a blind Flipt or any error.++They are separate because Beat stays the **sole scheduler for the whole PG rollout**:+pipelines already reach PG without this module (``execute_pipeline_task_v2`` →+``complete_execution`` → ``execute_workflow_async`` → ``resolve_transport``), so the PG+scheduler exists only to retire the Beat *deployment* — a later, deliberate step. While+the env gate is off this module writes **nothing at all**, so Beat's tables are untouched+for the entire rollout and turning the flag off again is a pure Flipt flip with nothing+to restore."""from __future__ import annotations@@ -37,6 +44,34 @@ logger = logging.getLogger(__name__)# buckets the %-rollout on pipeline_id (each subsystem keys on its own entity).+# Second gate, ahead of the Flipt flag: hand-over to the PG scheduler happens only+# when this is explicitly on. Referenced as "pg_scheduler_enabled" by this module's+# callers' comments (scheduler/helper.py, reconcile_pg_schedules) since the ramp was+# designed, but never actually implemented — resolve_schedule_owner gated on+# pg_queue_enabled alone, so the two flips were welded together.+#+# They must be separable, because Beat stays the SOLE scheduler for the whole PG+# rollout: pipelines already reach PG without it (execute_pipeline_task_v2 →+# complete_execution → execute_workflow_async → resolve_transport), so the PG+# scheduler exists only to retire the Beat *deployment* — a later, deliberate step.+#+# Without this gate, turning pg_queue_enabled on would hand schedules to a PG+# scheduler that is not running: reconcile_ownership_for disables the Beat+# PeriodicTask, nothing polls the PG side, and the pipeline has no firer at all. It+# runs in the backend on every schedule save, so scaling Beat to zero does not avoid+# it and no ramp command is needed to trigger it — a user saving a schedule suffices.+_PG_SCHEDULER_ENABLED_ENV = "PG_SCHEDULER_ENABLED"+++def pg_scheduler_enabled() -> bool:+ """Whether schedule hand-over to the PG scheduler is switched on at all.++ Defaults **off**. Flip it only alongside deploying the PG scheduler worker, as+ part of retiring Celery Beat.+ """+ return os.environ.get(_PG_SCHEDULER_ENABLED_ENV, "false").strip().lower() == "true"++def resolve_schedule_owner(pipeline_id: str, organization_id: str | None) -> bool:"""True → the PG scheduler owns this schedule; False → Celery Beat does.@@ -44,7 +79,13 @@ def resolve_schedule_owner(pipeline_id: str, organization_id: str | None) -> booflag, keyed on ``pipeline_id`` for a stable percentage bucket. **Fails closed toBeat** on a blind Flipt or any error — so a schedule never silently loses itsfirer.++ Requires ``PG_SCHEDULER_ENABLED`` **as well as** the flag: during the PG rollout+ the flag is on while the PG scheduler is still dark, and a schedule handed to a+ scheduler that is not running would simply stop firing."""+ if not pg_scheduler_enabled():+ return Falseif os.environ.get("FLIPT_SERVICE_AVAILABLE", "false").lower() != "true":logger.warning("resolve_schedule_owner: FLIPT_SERVICE_AVAILABLE != true "@@ -68,8 +109,7 @@ def resolve_schedule_owner(pipeline_id: str, organization_id: str | None) -> boo# edit — warn with traceback rather than logger.exception so a persistently# down Flipt doesn't bury real errors as a per-edit Sentry exception.logger.warning(- "resolve_schedule_owner: Flipt check failed for pipeline %s; "- "leaving on Beat",+ "resolve_schedule_owner: Flipt check failed for pipeline %s; leaving on Beat",pipeline_id,exc_info=True,)@@ -77,6 +117,27 @@ def resolve_schedule_owner(pipeline_id: str, organization_id: str | None) -> booreturn bool(owned)+def _has_stale_pg_ownership(pipeline_id: str) -> bool:+ """True if this row claims PG ownership while the gate is off — i.e. it needs+ releasing back to Beat.++ **Fails closed to today's behaviour**: on any DB error this returns False, so the+ caller takes the historical `return False` path and writes nothing. A repair that+ cannot confirm it is needed must not run.+ """+ try:+ return PgPeriodicSchedule.objects.filter(+ pipeline_id=pipeline_id, pg_owned=True+ ).exists()+ except Exception:+ logger.exception(+ "reconcile_ownership_for: could not check stale PG ownership for "+ "pipeline %s; leaving the schedule on its current firer",+ pipeline_id,+ )+ return False++def reconcile_ownership_for(pipeline_id: str, organization_id: str | None, *, active: bool) -> bool | None:@@ -94,16 +155,73 @@ def reconcile_ownership_for(Best-effort: a DB failure is logged and swallowed so it can never break thecaller. Returns the resolved ``pg_owned`` on success, or **None** if thetransaction failed (so the ramp command can tally + surface failures).++ **No-op while ``PG_SCHEDULER_ENABLED`` is off** — writing NEITHER table, *unless*+ the row contradicts the gate (see below). Returning early rather than relying on+ ``resolve_schedule_owner`` returning False matters: that path would still issue+ ``PeriodicTask.update(enabled=active)`` and bump ``PeriodicTasks.update_changed()``+ on every schedule save, forcing a Beat reload to write back a value Beat already+ had. Beat's tables stay untouched for the whole rollout, so flag-off is a pure+ Flipt flip with nothing to restore."""- pg_owned = resolve_schedule_owner(pipeline_id, organization_id)+ if not pg_scheduler_enabled():+ # Gate off ⇒ Beat owns everything. ASSERT that instead of assuming it.+ #+ # The original blanket `return False` assumed pg_owned could never be True+ # while the gate is off. An image predating this gate breaks that assumption:+ # there, resolve_schedule_owner keys on the Flipt flag ALONE, so flipping+ # pg_queue_enabled hands schedules to PG. Roll forward onto this gate and the+ # row is stranded — writing nothing can never correct it, and NO code path+ # (not even `reconcile_pg_schedules` without --mirror-only, which routes back+ # through here) can clear it. Only hand-written SQL could, which is not a+ # deploy procedure. Observed on integration 2026-08-12: Beat and the PG+ # scheduler both fired one pipeline 2.4s apart; a single execution resulted+ # only because the Celery consumers happened to be scaled to zero.+ #+ # This is reachable in production WITHOUT a mis-built image: deploy this+ # branch, roll back to a pre-gate build while the flag is on, roll forward.+ # Rollback is precisely when a stranded scheduler is least affordable.+ #+ # Cost in the normal case is one indexed existence check and still zero+ # writes, so the "Beat's tables stay untouched" guarantee holds for every+ # environment that never had a contradictory row.+ if not _has_stale_pg_ownership(pipeline_id):+ return False+ logger.warning(+ "reconcile_ownership_for: pipeline %s is pg_owned while "+ "PG_SCHEDULER_ENABLED is off (stale hand-over from a pre-gate build); "+ "releasing it back to Beat",+ pipeline_id,+ )+ pg_owned = False+ else:+ pg_owned = resolve_schedule_owner(pipeline_id, organization_id)try:with transaction.atomic():+ was_pg_owned = (+ PgPeriodicSchedule.objects.filter(pipeline_id=pipeline_id)+ .values_list("pg_owned", flat=True)+ .first()+ )# queryset .update() doesn't fire auto_now, so bump updated_at# explicitly (mirrors _mirror_periodic_schedule_set_enabled).updates: dict = {"pg_owned": pg_owned, "updated_at": timezone.now()}- if not pg_owned:- # Back on Beat → clear the PG next-run so a future re-hand-over- # baselines instead of firing immediately on a stale timestamp.+ # Baseline on an ownership CHANGE, in either direction. NULL means+ # "record a baseline next tick, don't fire this cycle"+ # (pg_queue/models.py:366), and that is the only thing standing between a+ # hand-over and an immediate unscheduled run: the tick selects+ # `WHERE pg_owned AND enabled AND (next_run_at IS NULL OR <= now())`, so a+ # next_run_at left over from an earlier PG-ownership period is already in+ # the past and fires at once. Observed on integration 2026-08-14 —+ # gallh_load_test carried next_run_at=2026-08-12 06:08 from a stale row and+ # fired ~2s after being re-enabled, two days late, alongside the operator's+ # manual run.+ #+ # Scoped to the TRANSITION, not every call: this runs on every pipeline+ # save, and clearing unconditionally would re-baseline mid-cycle — a save at+ # 12:07:59 against a 12:08 next_run_at would skip that fire entirely.+ handing_over_to_pg = pg_owned and not was_pg_owned+ if not pg_owned or handing_over_to_pg:updates["next_run_at"] = None# The mirror row exists from the dual-write / backfill; guard# anyway — a missing row means nothing to own yet.@@ -121,9 +239,51 @@ def reconcile_ownership_for(# Return False so the ramp count isn't inflated past what's live.return False# Beat owns it only when active AND not handed to PG.- PeriodicTask.objects.filter(name=pipeline_id).update(- enabled=active and not pg_owned- )+ beat_updates: dict = {"enabled": active and not pg_owned}+ # Baseline Beat's clock on the way BACK, for the same reason next_run_at+ # is baselined on the way out — and this half was missing.+ #+ # DatabaseScheduler keeps no next_run_at; it derives due-ness from+ # PeriodicTask.last_run_at against the crontab. A schedule that spent days+ # PG-owned carries a last_run_at from before the hand-over, so the instant+ # `enabled` flips back every missed interval is overdue and Beat replays+ # them at once. Observed on integration 2026-08-24: releasing 23 schedules+ # fired 4 pipelines plus 3 periodics within 30 ms of "Released to Beat".+ #+ # An earlier comment here asserted the opposite — that DatabaseScheduler+ # "recomputes due-ness from the crontab each tick, so re-enabling never+ # produced a catch-up run". That was wrong: it recomputes from last_run_at,+ # which is exactly what makes it catch up.+ #+ # Scoped to the RELEASE transition, mirroring the next_run_at rule above:+ # stamping it on every call would push the clock forward on an ordinary+ # pipeline save and silently skip a due fire.+ if was_pg_owned and not pg_owned:+ beat_updates["last_run_at"] = timezone.now()+ matched = PeriodicTask.objects.filter(name=pipeline_id).update(**beat_updates)+ if not matched:+ # Branch the WHOLE message, not just the verb. The two directions mean+ # opposite things, and an earlier version applied the alarming tail to+ # both — which is how the direction that matters gets filtered out.+ if pg_owned:+ # Adopt: nothing to switch off. The mirror row was just written+ # pg_owned=True and the PG tick selects WHERE pg_owned AND enabled,+ # so PG *is* the firer. Benign, and this path runs on every pipeline+ # save, so a warning here would be noise.+ logger.info(+ "reconcile_ownership_for: no Beat PeriodicTask named %s to "+ "disable — PG owns it now, nothing needed switching off",+ pipeline_id,+ )+ else:+ # Release: PG has let go and Beat has no row to take over, so the+ # schedule has no firer at all — on the rollback path. Silent before+ # this: a bulk .update() reports success by returning 0.+ logger.warning(+ "reconcile_ownership_for: no Beat PeriodicTask named %s to "+ "release to — the schedule may now have no firer",+ pipeline_id,+ )# Bulk .update() bypasses django-celery-beat's post_save signal, so# PeriodicTasks.last_update never bumps and DatabaseScheduler never# reloads — Beat would keep firing the schedule from its stalediff --git a/backend/scheduler/tasks.py b/backend/scheduler/tasks.pyindex 7db1cad53..755d878b2 100644--- a/backend/scheduler/tasks.py+++ b/backend/scheduler/tasks.py@@ -34,47 +34,86 @@ logger = logging.getLogger(__name__)# ----------------------------------------------------------------------------def _retargeted_next_run_at(pipeline_id: str, cron_string: str) -> datetime | None:- """The ``next_run_at`` a cron EDIT should set for Beat parity, or ``None`` to- leave it as the PG scheduler's baseline owns it.-- Returns a recomputed next-match ONLY when ``cron_string`` differs from an- already-baselined mirror row (``next_run_at`` set). Returns ``None`` for a- brand-new row (no mirror yet) and a not-yet-baselined one (``next_run_at``- NULL) so the scheduler's no-burst baseline records the first next-time.+# Sentinel: leave the ``next_run_at`` column alone entirely. Distinct from the+# value ``None``, which is a deliberate instruction to WRITE NULL — the PG tick+# reads NULL as "record a baseline next pass, don't fire this one". Conflating+# the two is what let a stale next_run_at survive a resume.+_LEAVE_NEXT_RUN_AT = object()+++def _next_run_at_for_upsert(+ pipeline_id: str, cron_string: str, enabled: bool+) -> datetime | None | object:+ """What the mirror upsert should do with ``next_run_at``. Three outcomes:++ * :data:`_LEAVE_NEXT_RUN_AT` — don't touch the column. A brand-new row, a+ not-yet-baselined one (``next_run_at`` NULL), or an unchanged cron on a+ row that was already enabled. The PG tick's no-burst baseline owns it.+ * ``None`` — write NULL, i.e. "baseline on the next tick, don't fire". A+ RESUME: the mirror row was ``enabled=False`` and the incoming state is+ ``True``.+ * a ``datetime`` — a cron EDIT on an already-baselined row: retarget to the+ new cron's next match for Beat parity (UN-3690), else the scheduler fires+ once more at the stale old-cron time.++ **Why resume needs the explicit NULL, and why it is checked first.** While a+ schedule is paused its ``next_run_at`` keeps drifting into the past, so the+ instant ``enabled`` flips back the tick's+ ``WHERE pg_owned AND enabled AND (next_run_at IS NULL OR <= now())`` matches+ and the pipeline runs immediately — days late, on top of whatever the+ operator triggered by hand. ``enable_task`` already baselines for exactly+ this reason (see ``_mirror_periodic_schedule_set_enabled``), but the UI+ resume does not take that path: it re-saves the pipeline, which reaches+ ``SchedulerHelper._schedule_task_job`` → here and ``reconcile_ownership_for``+ — and that one deliberately baselines only on a Beat→PG hand-over+ (``pg_owned and not was_pg_owned``), so an already-``pg_owned`` row keeps its+ stale value and nothing clears it. Observed on integration 2026-08-12, -14+ and -20: three enables of ``gallh_load_test``, three spurious runs 2-3s+ later, each settling back onto the cron afterwards. Exactly one extra run per+ enable — bounded, but it costs a full LLM pass over the source.++ Resume is checked BEFORE the cron comparison because a resume that also+ edited the cron must still baseline; ordering it the other way would let the+ ``cron_string == existing`` early-out swallow the resume case, which is the+ common one (a plain pause/resume does not change the cron).++ Scoped to the TRANSITION, not to every call — the same discipline+ ``reconcile_ownership_for`` uses. This runs on every pipeline save, so+ clearing unconditionally would re-baseline mid-cycle: a save at 12:07:59+ against a 12:08 ``next_run_at`` would skip that fire entirely.Fully guarded on purpose: this is an OPTIONAL enhancement over the mandatory``cron_string``/``enabled`` mirror write, so a read failure, a ``None``/invalidcron (normally rejected upstream by ``PipelineSerializer.validate_cron_string``- in ``pipeline_v2``), or a croniter error must degrade to ``None`` — never take- the base mirror write down with it.+ in ``pipeline_v2``), or a croniter error must degrade to leaving the column+ alone — never take the base mirror write down with it."""try:existing = (PgPeriodicSchedule.objects.filter(pipeline_id=pipeline_id)- .values("cron_string", "next_run_at")+ .values("cron_string", "next_run_at", "enabled").first())- if (- existing is None- or existing["next_run_at"] is None- or existing["cron_string"] == cron_string- ):+ if existing is None or existing["next_run_at"] is None:+ return _LEAVE_NEXT_RUN_AT+ if enabled and not existing["enabled"]:return None+ if existing["cron_string"] == cron_string:+ return _LEAVE_NEXT_RUN_ATreturn croniter(cron_string, timezone.now()).get_next(datetime)except Exception as exc:# Log the ACTUAL cause (bad read; cron=None → AttributeError;# CroniterBadCronError / CroniterBadDateError; a croniter API change) — a# generic "could not recompute" is a dead end when debugging a stale fire.logger.warning(- "pg_periodic_schedule: could not retarget next_run_at for pipeline %s "+ "pg_periodic_schedule: could not resolve next_run_at for pipeline %s ""(cron %r): %s",pipeline_id,cron_string,exc,exc_info=True,)- return None+ return _LEAVE_NEXT_RUN_ATdef mirror_periodic_schedule_upsert(@@ -94,18 +133,15 @@ def mirror_periodic_schedule_upsert("cron_string": cron_string,"enabled": enabled,}- # Beat parity: a cron EDIT on an already-baselined row must retarget- # next_run_at to the new cron's next match, else the PG scheduler fires- # once more at the STALE old-cron time — Beat has no such staleness, it- # recomputes due-ness live from the crontab each tick (UN-3690). Left- # untouched (NULL) for a brand-new row — the scheduler's no-burst baseline- # is correct there (a new schedule fires at its next match) — and for a- # Beat→PG hand-over, where next_run_at is already NULL (never set while- # Beat-owned; ownership clears it only on rollback to Beat, see- # reconcile_ownership_for). Computed via a fully-guarded helper so it can- # never take down the mandatory cron_string/enabled mirror write below.- next_run_at = _retargeted_next_run_at(pipeline_id, cron_string)- if next_run_at is not None:+ # next_run_at is owned by the PG tick, with two exceptions this path must+ # honour: a cron EDIT retargets it (Beat parity, UN-3690) and a RESUME+ # clears it to NULL so a value that went stale during the pause can't fire+ # a catch-up run the moment the schedule is re-enabled. The helper decides+ # which — including "leave it alone", which is NOT the same as writing NULL+ # — and is fully guarded so it can never take down the mandatory+ # cron_string/enabled mirror write below.+ next_run_at = _next_run_at_for_upsert(pipeline_id, cron_string, enabled)+ if next_run_at is not _LEAVE_NEXT_RUN_AT:defaults["next_run_at"] = next_run_atPgPeriodicSchedule.objects.update_or_create(pipeline_id=pipeline_id,@@ -123,8 +159,33 @@ def _mirror_periodic_schedule_set_enabled(pipeline_id: str, enabled: bool) -> No# Bump updated_at explicitly: queryset .update() does NOT trigger the# field's auto_now, so without this a pause/resume would change enabled# without advancing the "last changed" timestamp.+ updates: dict = {"enabled": enabled, "updated_at": timezone.now()}+ if enabled:+ # Resume → baseline instead of firing a stale next_run_at. While the+ # schedule was paused its next_run_at kept drifting into the past, so the+ # PG tick's `next_run_at <= now()` matches on the very next pass and the+ # pipeline runs IMMEDIATELY on resume — the longer the pause, the more+ # certain. NULL means "record a baseline next tick, don't fire this cycle"+ # (pg_queue/models.py:366), which resumes at the next cron match instead.+ #+ # Observed on integration 2026-08-14: gallh_load_test fired ~2s after+ # being re-enabled, against a next_run_at two days old.+ #+ # CORRECTION (2026-08-24): this used to add "Beat parity, not a new rule:+ # DatabaseScheduler ... recomputes due-ness from the crontab each tick, so+ # re-enabling never produced a catch-up run there." That was WRONG.+ # DatabaseScheduler recomputes from PeriodicTask.last_run_at, which is+ # exactly what makes it catch up too — releasing 23 schedules back to Beat+ # fired 4 pipelines and 3 periodics within 30 ms. Beat needs the same+ # baseline on its own clock; see scheduler/ownership.py. So this is a+ # shared hazard of both schedulers, not a PG-only regression.+ #+ # Safe to do unconditionally here: this helper is reached only from+ # enable_task/disable_task (an explicit pause/resume), never from the+ # per-save path — so it cannot re-baseline mid-cycle and skip a fire.+ updates["next_run_at"] = Nonematched = PgPeriodicSchedule.objects.filter(pipeline_id=pipeline_id).update(- enabled=enabled, updated_at=timezone.now()+ **updates)if matched == 0:# No mirror row — e.g. a pipeline scheduled before this shipped, ordiff --git a/backend/scheduler/tests/test_pg_periodic_schedule_mirror.py b/backend/scheduler/tests/test_pg_periodic_schedule_mirror.pyindex 9d094f9a9..8b4439f7a 100644--- a/backend/scheduler/tests/test_pg_periodic_schedule_mirror.py+++ b/backend/scheduler/tests/test_pg_periodic_schedule_mirror.py@@ -83,40 +83,49 @@ class TestUpsertMirror:)-class TestUpsertNextRunRecompute:- """UN-3690 — a cron EDIT must retarget ``next_run_at`` (Beat parity), so the- new time takes effect this cycle instead of the pipeline firing once more at- the stale old-cron time. Only for an already-baselined row: a fresh row and a- Beat→PG hand-over keep ``next_run_at`` NULL for the scheduler's no-burst- baseline.- """+class _UpsertNextRunHelpers:+ """Shared setup for the two ``next_run_at`` suites below. Helpers only — no+ tests live here, so neither suite re-runs the other's cases."""_OLD = datetime(2026, 1, 1, 9, 0, tzinfo=dt_timezone.utc)@staticmethod- def _mock_existing(sched, *, cron_string, next_run_at):+ def _mock_existing(sched, *, cron_string, next_run_at, enabled=True):_stub_existing_row(sched,Noneif cron_string is None- else {"cron_string": cron_string, "next_run_at": next_run_at},+ else {+ "cron_string": cron_string,+ "next_run_at": next_run_at,+ "enabled": enabled,+ },)@staticmethod- def _upsert(cron):+ def _upsert(cron, enabled=True):tasks.mirror_periodic_schedule_upsert(pipeline_id=_PIPELINE_ID,organization_id=_ORG,workflow_id=_WORKFLOW_ID,pipeline_name=_REAL_NAME,cron_string=cron,- enabled=True,+ enabled=enabled,)@staticmethoddef _defaults(sched):return sched.objects.update_or_create.call_args.kwargs["defaults"]++class TestUpsertNextRunRecompute(_UpsertNextRunHelpers):+ """UN-3690 — a cron EDIT must retarget ``next_run_at`` (Beat parity), so the+ new time takes effect this cycle instead of the pipeline firing once more at+ the stale old-cron time. Only for an already-baselined row: a fresh row and a+ Beat→PG hand-over keep ``next_run_at`` NULL for the scheduler's no-burst+ baseline.+ """+def test_cron_change_on_baselined_row_retargets_next_run_at(self):now = datetime(2026, 1, 1, 8, 0, tzinfo=dt_timezone.utc)with (@@ -172,6 +181,86 @@ class TestUpsertNextRunRecompute:assert "next_run_at" not in self._defaults(sched)+class TestUpsertResumeBaselinesStaleNextRun(_UpsertNextRunHelpers):+ """A UI resume must not fire a catch-up run.++ ``enable_task`` baselines ``next_run_at`` on resume, but the UI does not take+ that path — it re-saves the pipeline, reaching+ ``SchedulerHelper._schedule_task_job`` → ``mirror_periodic_schedule_upsert``+ (here) and ``reconcile_ownership_for``. The latter baselines only on a+ Beat→PG hand-over, so a row that is ALREADY ``pg_owned`` keeps whatever+ ``next_run_at`` it had when it was paused — days in the past — and the PG+ tick's ``next_run_at <= now()`` fires it on the very next pass.++ Observed three times on integration (2026-08-12, -14, -20): each enable of+ ``gallh_load_test`` produced a spurious run 2-3s later, then settled onto the+ cron correctly.+ """++ def test_resume_clears_a_stale_next_run_at(self):+ """The bug, directly: paused row + past next_run_at, re-enabled."""+ with patch("scheduler.tasks.PgPeriodicSchedule") as sched:+ self._mock_existing(+ sched, cron_string="8 * * * *", next_run_at=self._OLD, enabled=False+ )+ self._upsert("8 * * * *", enabled=True)+ defaults = self._defaults(sched)+ # Present AND None: writing NULL is the baseline instruction. Merely+ # omitting the key would leave the stale value in place — the bug.+ assert "next_run_at" in defaults+ assert defaults["next_run_at"] is None++ def test_resume_that_also_edits_the_cron_still_baselines(self):+ """Resume is checked before the cron comparison, so a combined+ resume+edit baselines rather than retargeting. Both are non-firing, but+ this pins the ordering — reversing it would let the equal-cron early-out+ swallow the common plain-resume case."""+ with patch("scheduler.tasks.PgPeriodicSchedule") as sched:+ self._mock_existing(+ sched, cron_string="0 9 * * *", next_run_at=self._OLD, enabled=False+ )+ self._upsert("0 10 * * *", enabled=True)+ assert self._defaults(sched)["next_run_at"] is None++ def test_a_save_on_an_already_enabled_row_is_untouched(self):+ """The mid-cycle guard. This helper runs on EVERY pipeline save, so a+ rename at 12:07:59 against a 12:08 next_run_at must not re-baseline and+ skip that fire. Only the False→True transition clears."""+ with patch("scheduler.tasks.PgPeriodicSchedule") as sched:+ self._mock_existing(+ sched, cron_string="8 * * * *", next_run_at=self._OLD, enabled=True+ )+ self._upsert("8 * * * *", enabled=True)+ assert "next_run_at" not in self._defaults(sched)++ def test_pause_does_not_clear_next_run_at(self):+ """Nothing fires while disabled, so there is nothing to guard against —+ and clearing here would discard the value resume needs to detect."""+ with patch("scheduler.tasks.PgPeriodicSchedule") as sched:+ self._mock_existing(+ sched, cron_string="8 * * * *", next_run_at=self._OLD, enabled=True+ )+ self._upsert("8 * * * *", enabled=False)+ assert "next_run_at" not in self._defaults(sched)++ def test_resume_of_a_never_baselined_row_writes_nothing(self):+ """next_run_at already NULL is already the baseline — no write needed,+ and the read short-circuits before the enabled check."""+ with patch("scheduler.tasks.PgPeriodicSchedule") as sched:+ self._mock_existing(+ sched, cron_string="8 * * * *", next_run_at=None, enabled=False+ )+ self._upsert("8 * * * *", enabled=True)+ assert "next_run_at" not in self._defaults(sched)++ def test_the_sentinel_is_not_none(self):+ """The whole fix rests on 'leave alone' being distinguishable from 'write+ NULL'. If the sentinel were None-y, the resume baseline would silently+ degrade back to the old skip-the-write behaviour and every test above+ would still pass on the omission path."""+ assert tasks._LEAVE_NEXT_RUN_AT is not None++class TestHelperWiringSourcesRealName:"""The High contract: the mirror must store the user-facing pipeline name,NOT the synthetic ``"Pipeline job-<id>"`` label that the PeriodicTask args@@ -336,3 +425,53 @@ class TestDeleteMirror:pt.objects.get.return_value = MagicMock()sched.objects.filter.return_value.delete.side_effect = RuntimeError("db down")tasks.delete_periodic_task(_PIPELINE_ID) # must not raise+++class TestResumeBaselinesInsteadOfCatchingUp:+ """Re-enabling a paused schedule must resume at the next cron match, not fire now.++ While a schedule is paused its `next_run_at` keeps drifting into the past, so the+ PG tick's `next_run_at <= now()` matches on the very next pass and the pipeline runs+ IMMEDIATELY on resume — the longer the pause, the more certain. NULL is the guard:+ "record a baseline next tick, don't fire this cycle" (pg_queue/models.py:366).++ Observed on integration 2026-08-14: gallh_load_test fired ~2s after being+ re-enabled, against a next_run_at two days old, on top of the operator's manual run.++ CORRECTION (2026-08-24): this docstring used to claim "Beat parity, not a new rule —+ DatabaseScheduler ... recomputes due-ness from the crontab each tick, so resume+ never produced a catch-up run there." Wrong on the mechanism: it recomputes from+ PeriodicTask.last_run_at, so Beat catches up as well. Both schedulers need their+ clock baselined on a hand-over; the Beat half is pinned by+ scheduler/tests/test_pg_schedule_ownership.py::TestBeatClockBaselineOnRelease.+ """++ def _run(self, fn):+ with (+ patch("scheduler.tasks.PeriodicTask") as pt,+ patch("scheduler.tasks.PeriodicTasks"),+ patch("scheduler.tasks.PipelineProcessor"),+ patch("scheduler.tasks.PgPeriodicSchedule") as sched,+ patch("scheduler.tasks.transaction.atomic", return_value=nullcontext()),+ ):+ pt.objects.get.return_value = MagicMock()+ (+ sched.objects.select_for_update.return_value.filter.return_value+ .values_list.return_value.first.return_value+ ) = False+ sched.objects.filter.return_value.update.return_value = 1+ fn(_PIPELINE_ID)+ return sched.objects.filter.return_value.update.call_args.kwargs++ def test_resume_clears_next_run_at(self):+ kwargs = self._run(tasks.enable_task)+ assert kwargs["enabled"] is True+ assert kwargs["next_run_at"] is None++ def test_pause_leaves_next_run_at_alone(self):+ """Nothing fires while disabled, so there is nothing to guard against — and+ clearing here would discard the value resume needs to baseline against.+ """+ kwargs = self._run(tasks.disable_task)+ assert kwargs["enabled"] is False+ assert "next_run_at" not in kwargsdiff --git a/backend/scheduler/tests/test_pg_schedule_ownership.py b/backend/scheduler/tests/test_pg_schedule_ownership.pyindex bb4c1af48..4f05e8d15 100644--- a/backend/scheduler/tests/test_pg_schedule_ownership.py+++ b/backend/scheduler/tests/test_pg_schedule_ownership.py@@ -18,6 +18,16 @@ _ORG = "org_abc"class TestResolveScheduleOwner:+ @pytest.fixture(autouse=True)+ def _scheduler_gate_on(self, monkeypatch):+ """These pin the FINAL-phase behaviour, when hand-over is switched on.++ PG_SCHEDULER_ENABLED defaults off for the whole rollout (Beat stays the sole+ scheduler), so without this every case below would short-circuit to Beat and+ assert nothing about the logic it was written for.+ """+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")+def test_flipt_unavailable_is_beat(self, monkeypatch):monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "false")with patch("scheduler.ownership.check_feature_flag_status") as flag:@@ -52,6 +62,16 @@ class TestResolveScheduleOwner:class TestReconcileOwnership:+ @pytest.fixture(autouse=True)+ def _scheduler_gate_on(self, monkeypatch):+ """These pin the FINAL-phase behaviour, when hand-over is switched on.++ PG_SCHEDULER_ENABLED defaults off for the whole rollout (Beat stays the sole+ scheduler), so without this every case below would short-circuit to Beat and+ assert nothing about the logic it was written for.+ """+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")+@pytest.fixture(autouse=True)def _mock_periodic_tasks(self):# reconcile now bumps PeriodicTasks.last_update after the bulk .update();@@ -104,18 +124,26 @@ class TestReconcileOwnership:PT.objects.filter.return_value.update.call_args.kwargs["enabled"] is True)- def test_pg_owned_does_not_clear_next_run(self):+ def test_an_ALREADY_pg_owned_schedule_does_not_clear_next_run(self):+ """Narrowed: this is the True→True case, not "adopt never clears".++ Hand-over now DOES baseline (see TestNextRunBaselineOnTransition) — a+ next_run_at surviving adoption fires the pipeline immediately. What must not+ happen is re-baselining a schedule that was already PG-owned: reconcile runs on+ every pipeline save, so a save at 12:07:59 would push a 12:08 next_run_at out to+ 13:08 and skip that fire.++ `was_pg_owned` is set explicitly — a bare MagicMock is truthy, so this would+ otherwise pass for the wrong reason and keep passing if the scoping regressed.+ """sched, pt, resolve, txn = self._patches(owner=True)with sched as Sched, pt, resolve, txn:- Sched.objects.filter.return_value.update.return_value = 1+ qs = Sched.objects.filter.return_value+ qs.values_list.return_value.first.return_value = True+ qs.update.return_value = 1ownership.reconcile_ownership_for(_PID, _ORG, active=True)- # An active PG-owned schedule must NOT have its next_run_at reset (that- # would re-baseline and skip a fire).- assert (- "next_run_at"- not in Sched.objects.filter.return_value.update.call_args.kwargs- )+ assert "next_run_at" not in qs.update.call_args.kwargsdef test_paused_pipeline_keeps_beat_disabled_even_if_not_pg_owned(self):sched, pt, resolve, txn = self._patches(owner=False)@@ -167,6 +195,172 @@ class TestReconcileOwnership:_mock_periodic_tasks.update_changed.assert_not_called()+class TestPgSchedulerGate:+ """The gate that keeps Beat the sole scheduler for the whole PG rollout.++ Pipelines already reach PG without the PG scheduler (execute_pipeline_task_v2 →+ complete_execution → execute_workflow_async → resolve_transport), so the PG+ scheduler exists only to retire the Beat *deployment* — a later, deliberate step.+ Handing a schedule over before then leaves it with NO firer: Beat's PeriodicTask+ disabled, nothing polling the PG side.+ """++ def test_defaults_off(self, monkeypatch):+ monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ assert ownership.pg_scheduler_enabled() is False++ @pytest.mark.parametrize("value", ["false", "False", "0", "", "yes", "TRUE "])+ def test_only_an_exact_true_opts_in(self, monkeypatch, value):+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", value)+ assert ownership.pg_scheduler_enabled() is (value.strip().lower() == "true")++ def test_owner_stays_beat_even_with_the_flag_on(self, monkeypatch):+ """The rollout state: pg_queue_enabled on, PG scheduler still dark."""+ monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")+ with patch(+ "scheduler.ownership.check_feature_flag_status", return_value=True+ ) as flag:+ assert ownership.resolve_schedule_owner(_PID, _ORG) is False+ # Short-circuits before Flipt — no evaluation on every schedule save.+ flag.assert_not_called()++ def test_reconcile_writes_NEITHER_beat_table(self, monkeypatch):+ """The regression this gate exists to prevent.++ Returning early matters over merely resolving to Beat: that path would still+ issue PeriodicTask.update(enabled=active) and bump PeriodicTasks.update_changed()+ on every schedule save — writing back a value Beat already had and forcing a+ reload, on a table we promised not to touch.+ """+ monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")+ with (+ patch("scheduler.ownership.PeriodicTask") as PT,+ patch("scheduler.ownership.PeriodicTasks") as PTs,+ patch("scheduler.ownership.PgPeriodicSchedule") as Sched,+ patch("scheduler.ownership.check_feature_flag_status", return_value=True),+ ):+ # A CLEAN row: not pg_owned, so there is nothing to repair and the+ # write-free path must be taken. Must be set explicitly — a bare+ # MagicMock's .exists() is truthy, which would look like a stale row.+ Sched.objects.filter.return_value.exists.return_value = False+ assert ownership.reconcile_ownership_for(_PID, _ORG, active=True) is False++ PT.objects.filter.assert_not_called()+ PTs.update_changed.assert_not_called()+ # pg_owned is not WRITTEN either — otherwise switching the PG scheduler on+ # later would fire every already-owned schedule while Beat still fires it too.+ # (.filter() is now called once, for the read-only staleness check.)+ Sched.objects.filter.return_value.update.assert_not_called()+++class TestStalePgOwnershipIsReleased:+ """A row left ``pg_owned`` by a build predating the gate must be repairable.++ Reproduced on integration 2026-08-12: `snapshot.993`'s backend came from OSS+ `main`, where ``resolve_schedule_owner`` keys on the ``pg_queue_enabled`` Flipt+ flag ALONE. Turning the flag on handed one pipeline to PG. Rolling forward onto+ the gate then STRANDED it: the gate-off path wrote nothing, so nothing —+ including ``reconcile_pg_schedules``, which routes back through here — could+ clear it. Beat and the PG scheduler both fired that pipeline 2.4s apart; only+ one execution resulted because the Celery consumers happened to be at zero.++ Reachable in production without a mis-built image: deploy this branch, roll back+ to a pre-gate build while the flag is on, roll forward.+ """++ def _mocks(self, stale: bool):+ sched = patch("scheduler.ownership.PgPeriodicSchedule").start()+ sched.objects.filter.return_value.exists.return_value = stale+ sched.objects.filter.return_value.update.return_value = 1+ # These tests are DB-free, but the repair path (unlike the write-free one)+ # enters the real transaction.atomic() — same no-op stand-in as above.+ patch(+ "scheduler.ownership.transaction.atomic",+ return_value=contextlib.nullcontext(),+ ).start()+ return sched++ def teardown_method(self):+ patch.stopall()++ def test_a_stale_row_is_handed_back_to_beat(self, monkeypatch):+ monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ sched = self._mocks(stale=True)+ with (+ patch("scheduler.ownership.PeriodicTask") as PT,+ patch("scheduler.ownership.PeriodicTasks") as PTs,+ ):+ assert ownership.reconcile_ownership_for(_PID, _ORG, active=True) is False++ # pg_owned cleared, and next_run_at reset so a later re-hand-over baselines+ # instead of firing immediately on a stale timestamp.+ updates = sched.objects.filter.return_value.update.call_args.kwargs+ assert updates["pg_owned"] is False+ assert updates["next_run_at"] is None+ # Beat re-enabled in the same breath — releasing pg_owned without this would+ # leave the schedule with NO firer at all.+ #+ # UN-3796 (2026-08-24): the write now also baselines Beat's clock, so this+ # asserts per-key rather than on the whole kwargs dict. Not a loosening — the+ # repair path is a RELEASE, and it is the one most likely to hand back a row+ # whose last_run_at is deeply stale, which is exactly what makes Beat replay+ # every missed interval at once. TestBeatClockBaselineOnRelease pins the rule.+ beat = PT.objects.filter.return_value.update.call_args.kwargs+ assert beat["enabled"] is True+ assert "last_run_at" in beat+ # ...and Beat told to reload, else it keeps its stale in-memory copy.+ PTs.update_changed.assert_called_once()++ def test_a_paused_pipeline_is_not_resurrected_by_the_repair(self, monkeypatch):+ """Repair restores the FIRER, never the on/off state the user chose."""+ monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ self._mocks(stale=True)+ with (+ patch("scheduler.ownership.PeriodicTask") as PT,+ patch("scheduler.ownership.PeriodicTasks"),+ ):+ ownership.reconcile_ownership_for(_PID, _ORG, active=False)+ beat = PT.objects.filter.return_value.update.call_args.kwargs+ assert beat["enabled"] is False+ # Baselined even though it stays paused: if it is resumed later, Beat must not+ # then replay the backlog accrued while PG owned it.+ assert "last_run_at" in beat++ def test_flipt_is_never_consulted_while_the_gate_is_off(self, monkeypatch):+ """The repair resolves to Beat unconditionally.++ Asking Flipt could return True and re-hand the schedule to a PG scheduler+ that is not running — turning a double-fire into no firer at all.+ """+ monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")+ self._mocks(stale=True)+ with (+ patch("scheduler.ownership.PeriodicTask"),+ patch("scheduler.ownership.PeriodicTasks"),+ patch(+ "scheduler.ownership.check_feature_flag_status", return_value=True+ ) as flag,+ ):+ ownership.reconcile_ownership_for(_PID, _ORG, active=True)+ flag.assert_not_called()++ def test_a_db_error_on_the_check_falls_back_to_writing_nothing(self, monkeypatch):+ """A repair that cannot confirm it is needed must not run."""+ monkeypatch.delenv("PG_SCHEDULER_ENABLED", raising=False)+ sched = patch("scheduler.ownership.PgPeriodicSchedule").start()+ sched.objects.filter.side_effect = Exception("db down")+ with (+ patch("scheduler.ownership.PeriodicTask") as PT,+ patch("scheduler.ownership.PeriodicTasks") as PTs,+ ):+ assert ownership.reconcile_ownership_for(_PID, _ORG, active=True) is False+ PT.objects.filter.assert_not_called()+ PTs.update_changed.assert_not_called()++class TestReconcileAtomicityRealDB:"""The load-bearing invariant: the pg_owned write and the PeriodicTask writeare ONE transaction — if the PeriodicTask update fails, pg_owned rolls back@@ -227,3 +421,138 @@ class TestReconcileAtomicityRealDB:finally:RealPeriodicTask.objects.filter(name=pid).delete()PgPeriodicSchedule.objects.filter(pipeline_id=pid).delete()+++class TestNextRunBaselineOnTransition:+ """An ownership or on/off change must never cause an immediate unscheduled run.++ The PG tick selects `WHERE pg_owned AND enabled AND (next_run_at IS NULL OR+ next_run_at <= now())`. A next_run_at left over from an earlier PG-ownership period+ is already in the past, so the row fires on the very next pass. NULL is the guard —+ "record a baseline next tick, don't fire this cycle" (pg_queue/models.py:366).++ Observed on integration 2026-08-14: gallh_load_test carried+ next_run_at=2026-08-12 06:08 and fired ~2s after being re-enabled — two days late,+ on top of the operator's own manual run.++ CORRECTION (2026-08-24): this docstring used to add "Beat never did this:+ DatabaseScheduler keeps no persisted next_run_at and recomputes due-ness from the+ crontab each tick." That is wrong. It recomputes from ``PeriodicTask.last_run_at``,+ which is precisely why it DOES catch up — releasing the fleet fired four pipelines+ and three periodics inside 30 ms. Beat has the same failure mode from the other+ direction; see TestBeatClockBaselineOnRelease below.+ """++ def _reconcile(self, monkeypatch, *, was_pg_owned, now_pg_owned):+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")+ monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")+ with (+ patch("scheduler.ownership.PgPeriodicSchedule") as Sched,+ patch("scheduler.ownership.PeriodicTask"),+ patch("scheduler.ownership.PeriodicTasks"),+ patch(+ "scheduler.ownership.transaction.atomic",+ return_value=contextlib.nullcontext(),+ ),+ patch(+ "scheduler.ownership.check_feature_flag_status",+ return_value=now_pg_owned,+ ),+ ):+ qs = Sched.objects.filter.return_value+ qs.values_list.return_value.first.return_value = was_pg_owned+ qs.update.return_value = 1+ ownership.reconcile_ownership_for(_PID, _ORG, active=True)+ return qs.update.call_args.kwargs++ def test_handing_over_to_pg_baselines(self, monkeypatch):+ # The bug: a stale next_run_at surviving adoption fires the pipeline at once.+ updates = self._reconcile(monkeypatch, was_pg_owned=False, now_pg_owned=True)+ assert updates["pg_owned"] is True+ assert updates["next_run_at"] is None++ def test_releasing_to_beat_still_baselines(self, monkeypatch):+ updates = self._reconcile(monkeypatch, was_pg_owned=True, now_pg_owned=False)+ assert updates["next_run_at"] is None++ def test_an_unchanged_owner_is_NOT_re_baselined(self, monkeypatch):+ """The reason this is scoped to the transition.++ reconcile runs on every pipeline save. Clearing unconditionally would let a+ save at 12:07:59 re-baseline a 12:08 next_run_at to 13:08 — silently skipping+ that fire. Idempotent re-runs (every deploy) must leave the schedule alone.+ """+ updates = self._reconcile(monkeypatch, was_pg_owned=True, now_pg_owned=True)+ assert "next_run_at" not in updates+++class TestBeatClockBaselineOnRelease:+ """Handing a schedule BACK to Beat must reset Beat's clock, or Beat replays.++ The symmetric half of the class above, and the one that was missing.+ DatabaseScheduler stores no next_run_at; it derives due-ness from+ ``PeriodicTask.last_run_at`` against the crontab. A pipeline that spent days+ PG-owned still carries the last_run_at from before the hand-over, so the instant+ ``enabled`` flips back it is overdue by every interval it missed — and Beat fires+ them all at once.++ Observed on integration 2026-08-24: `converge_pg_scheduler` released 23 schedules+ and Beat dispatched four pipelines plus three dashboard_metrics.* periodics within+ 30 ms of logging "Released to Beat".+ """++ def _reconcile(self, monkeypatch, *, was_pg_owned, now_pg_owned, active=True):+ monkeypatch.setenv("PG_SCHEDULER_ENABLED", "true")+ monkeypatch.setenv("FLIPT_SERVICE_AVAILABLE", "true")+ with (+ patch("scheduler.ownership.PgPeriodicSchedule") as Sched,+ patch("scheduler.ownership.PeriodicTask") as Beat,+ patch("scheduler.ownership.PeriodicTasks"),+ patch(+ "scheduler.ownership.transaction.atomic",+ return_value=contextlib.nullcontext(),+ ),+ patch(+ "scheduler.ownership.check_feature_flag_status",+ return_value=now_pg_owned,+ ),+ ):+ qs = Sched.objects.filter.return_value+ qs.values_list.return_value.first.return_value = was_pg_owned+ qs.update.return_value = 1+ ownership.reconcile_ownership_for(_PID, _ORG, active=active)+ return Beat.objects.filter.return_value.update.call_args.kwargs++ def test_releasing_to_beat_stamps_last_run_at(self, monkeypatch):+ beat = self._reconcile(monkeypatch, was_pg_owned=True, now_pg_owned=False)+ assert beat["enabled"] is True+ assert "last_run_at" in beat, (+ "release must baseline Beat's clock, else every interval missed while "+ "PG owned the schedule is overdue and Beat replays them at once"+ )+ assert beat["last_run_at"] is not None++ def test_handing_over_to_pg_does_NOT_touch_beats_clock(self, monkeypatch):+ """Adoption switches Beat OFF, so its clock is irrelevant — and overwriting it+ would destroy the value the eventual release needs to restore from."""+ beat = self._reconcile(monkeypatch, was_pg_owned=False, now_pg_owned=True)+ assert beat["enabled"] is False+ assert "last_run_at" not in beat++ def test_an_unchanged_owner_is_NOT_re_stamped(self, monkeypatch):+ """Same transition-scoping reason as next_run_at: reconcile runs on every+ pipeline save, and stamping unconditionally would push Beat's clock forward+ on an ordinary edit and silently skip a due fire."""+ beat = self._reconcile(monkeypatch, was_pg_owned=False, now_pg_owned=False)+ assert "last_run_at" not in beat++ def test_a_paused_pipeline_is_released_disabled_but_still_baselined(+ self, monkeypatch+ ):+ """A paused schedule comes back paused — but if it is later resumed, Beat must+ not then replay the backlog it accrued while PG owned it."""+ beat = self._reconcile(+ monkeypatch, was_pg_owned=True, now_pg_owned=False, active=False+ )+ assert beat["enabled"] is False+ assert "last_run_at" in beatdiff --git a/backend/workflow_manager/execution/tests/test_pg_finalization_fixes.py b/backend/workflow_manager/execution/tests/test_pg_finalization_fixes.pyindex c3d725216..d9cc5ba4d 100644--- a/backend/workflow_manager/execution/tests/test_pg_finalization_fixes.py+++ b/backend/workflow_manager/execution/tests/test_pg_finalization_fixes.py@@ -22,9 +22,20 @@ from workflow_manager.workflow_v2.models.workflow import Workflowdef _age(execution, seconds):- """Backdate modified_at (auto_now) via a direct UPDATE that bypasses auto_now."""- WorkflowExecution.objects.filter(pk=execution.pk).update(- modified_at=timezone.now() - timedelta(seconds=seconds)+ """Backdate modified_at (auto_now) via a direct UPDATE that bypasses auto_now.++ Ages the FILE rows too, because that is where ``recover_stuck_pg_executions``+ measures staleness: the execution row's ``modified_at`` is effectively its start+ time (file completions write the file row, not the execution row), so ageing only+ the execution would describe a run that started long ago and finished a moment+ ago — a live execution mid-callback, which the endpoint must NOT touch. A real+ strand has quiet files, and that is what this reproduces. Tests that specifically+ need the two timestamps to diverge override the file rows after calling this.+ """+ stale = timezone.now() - timedelta(seconds=seconds)+ WorkflowExecution.objects.filter(pk=execution.pk).update(modified_at=stale)+ WorkflowFileExecution.objects.filter(workflow_execution=execution).update(+ modified_at=stale)@@ -46,9 +57,9 @@ class RecoverStuckPgExecutionsTests(TestCase):)return ex- def _call(self, stuck_seconds=60):+ def _call(self, stuck_seconds=60, limit=100):req = MagicMock()- req.data = {"stuck_seconds": stuck_seconds, "limit": 100}+ req.data = {"stuck_seconds": stuck_seconds, "limit": limit}return self.view.recover_stuck_pg_executions(req).datadef test_all_files_completed_recovers_to_completed(self):@@ -102,28 +113,81 @@ class RecoverStuckPgExecutionsTests(TestCase):assert out["scanned"] == 0assert ex.status == ExecutionStatus.EXECUTING.value- def test_file_less_stuck_is_skipped_not_failed(self):- # A file-less PG exec may still be QUEUED (a backlog/outage can outlast the- # stuck window) — it must NOT be failed, so a delayed worker can still- # finalize it (the one-way guard would otherwise block recovery).+ def test_file_less_stuck_is_left_alone_and_no_longer_consumes_the_window(self):+ """A file-less PG exec may still be QUEUED (a backlog/outage can outlast the+ stuck window) — it must NOT be failed, so a delayed worker can still finalize+ it (the one-way guard would otherwise block recovery). That invariant is+ unchanged and is what the status assertion below pins.++ The COUNTER expectation is deliberately updated: this used to assert+ ``skipped == 1``, i.e. the row was selected and then rejected downstream.+ It is now filtered out at selection instead, so it is never scanned. The+ mechanism moved because a downstream skip leaves ``modified_at`` untouched,+ so these rows were re-selected on every sweep forever and crowded genuinely+ recoverable executions out of the window — see the starvation test below.+ """ex = self._exec(ExecutionStatus.PENDING, files=[])_age(ex, 9999)out = self._call()ex.refresh_from_db()- assert out["skipped"] == 1+ assert out["scanned"] == 0+ assert out["skipped"] == 0+ assert out["failed"] == 0assert ex.status == ExecutionStatus.PENDING.value- def test_celery_execution_never_scanned(self):+ def test_celery_execution_IS_now_recovered(self):+ """INVERTED deliberately (UN-3796). This asserted `scanned == 0` — that a+ Celery execution is never touched.++ That exclusion is the bug. A PG cutover removes the Celery workers, and an+ execution whose files all finished but whose chord callback never fired then+ sits EXECUTING forever: the callback is gone, there is no pg_barrier_state to+ recover from, and this endpoint refused to look. Nothing else could close it.++ The finalization logic was always transport-agnostic — it reads file statuses+ and recomputes the terminal status. Only the selection was narrow.+ """ex = self._exec(ExecutionStatus.EXECUTING, pg=False, files=[ExecutionStatus.COMPLETED])_age(ex, 9999)out = self._call()ex.refresh_from_db()- assert out["scanned"] == 0 # queue_message_id IS NULL → PG filter excludes it+ assert out["recovered"] == 1+ assert ex.status == ExecutionStatus.COMPLETED.value++ def test_a_NEVER_DISPATCHED_execution_is_left_to_the_undispatched_sweep(self):+ """The disjointness guard. Both handles NULL means the request died before+ dispatch — undispatched_sweep.py's row, which marks it ERROR with "did not+ start, safe to re-run".++ Both sweeps run on the same reaper cadence against the same table, so an+ overlap would be live from the first tick: one marking it ERROR while the+ other finalized it from whatever files happened to exist.+ """+ ex = WorkflowExecution.objects.create(+ workflow=self.wf,+ status=ExecutionStatus.EXECUTING.value,+ queue_message_id=None,+ task_id=None,+ )+ WorkflowFileExecution.objects.create(+ workflow_execution=ex, file_name="f", status=ExecutionStatus.COMPLETED.value+ )+ _age(ex, 9999)+ out = self._call()+ ex.refresh_from_db()+ assert out["scanned"] == 0assert ex.status == ExecutionStatus.EXECUTING.value- def test_still_processing_is_skipped(self):+ def test_still_processing_is_never_finalized(self):+ """A non-terminal file means work may still be in flight — finalizing would+ terminalize a live execution, and the one-way guard then blocks any correction.+ That invariant is unchanged.++ Counter expectation updated for the same reason as the file-less case: the row+ is now excluded at selection rather than skipped after being scanned.+ """ex = self._exec(ExecutionStatus.EXECUTING,files=[ExecutionStatus.COMPLETED, ExecutionStatus.EXECUTING],@@ -131,7 +195,8 @@ class RecoverStuckPgExecutionsTests(TestCase):_age(ex, 9999)out = self._call()ex.refresh_from_db()- assert out["skipped"] == 1+ assert out["scanned"] == 0+ assert out["failed"] == 0assert ex.status == ExecutionStatus.EXECUTING.valuedef test_recently_modified_not_recovered(self):@@ -141,7 +206,160 @@ class RecoverStuckPgExecutionsTests(TestCase):assert out["scanned"] == 0assert ex.status == ExecutionStatus.EXECUTING.value+ def test_backlog_of_unrecoverable_rows_does_not_starve_a_recoverable_one(self):+ """THE regression. Selection is oldest-first with a hard ``limit``, so any row+ that is selected but never finalized occupies a slot on every future sweep —+ a skip does not advance ``modified_at``, so the same rows come back forever.+ A recoverable execution behind them is never reached: not scanned, not+ skipped, invisible.++ Found live during the UN-3796 cutover rehearsal: 1964 candidates, 1122 of them+ permanently unrecoverable (476 file-less, 646 with a non-terminal file), and a+ freshly stranded execution at rank 1964 that every sweep failed to see while+ logging a healthy-looking "scanned 100, recovered 0".++ Mutation check: drop either ``Exists`` clause from the selection query and this+ fails — the backlog fills the window and ``recovered`` is 0.+ """+ limit = 3+ # Older than the victim, and permanently unrecoverable: the two shapes that+ # made up the real backlog.+ for i in range(limit * 2):+ junk = self._exec(+ ExecutionStatus.EXECUTING,+ files=[] if i % 2 else [ExecutionStatus.EXECUTING],+ )+ _age(junk, 9000)+ victim = self._exec(ExecutionStatus.EXECUTING, files=[ExecutionStatus.COMPLETED])+ _age(victim, 100) # NEWEST → last in oldest-first order++ out = self._call(limit=limit)++ victim.refresh_from_db()+ assert out["recovered"] == 1+ assert victim.status == ExecutionStatus.COMPLETED.value++ def test_window_is_spent_only_on_rows_that_can_finalize(self):+ """The drain property that makes oldest-first safe: everything scanned is+ finalized, so it leaves the candidate set and the backlog shrinks. If skips+ can consume the window the queue stops moving and FIFO stops being fair.+ """+ for _ in range(5):+ _age(self._exec(ExecutionStatus.EXECUTING, files=[]), 9000)+ _age(+ self._exec(ExecutionStatus.EXECUTING, files=[ExecutionStatus.EXECUTING]),+ 9000,+ )+ _age(+ self._exec(ExecutionStatus.EXECUTING, files=[ExecutionStatus.COMPLETED]), 9000+ )++ out = self._call()++ assert out["scanned"] == 1+ assert out["recovered"] == 1+ assert out["skipped"] == 0++ def test_execution_time_comes_from_the_last_file_not_from_now(self):+ """``update_execution()`` stamps ``execution_time = now - created_at`` on any+ terminal transition. Correct for a live finalization; wrong here, where the+ sweep may finalize work that ended long ago — it would record how late the+ reaper was, not how long the run took. The integration backlog would have+ written multi-day runtimes onto executions that ran for minutes.+ """+ ex = self._exec(ExecutionStatus.EXECUTING, files=[ExecutionStatus.COMPLETED])+ _age(ex, 9000)+ # Set the timestamps AFTER _age — it backdates the file rows too, so doing this+ # first would have them overwritten and the assertion would read 0.0.+ created = timezone.now() - timedelta(seconds=9000)+ # modified_at MUST be passed explicitly here. BaseModelManager.update() does+ # `kwargs.setdefault("modified_at", timezone.now())`, so updating created_at+ # alone silently re-stamps modified_at to NOW and un-ages the row _age() just+ # aged — the execution then fails `modified_at__lt=cutoff`, the endpoint reports+ # scanned=0, and the test fails on STATUS with no hint that a timestamp moved.+ # That cost a CI round trip; the manager's docstring documents the override.+ WorkflowExecution.objects.filter(pk=ex.pk).update(+ created_at=created,+ modified_at=timezone.now() - timedelta(seconds=9000),+ )+ WorkflowFileExecution.objects.filter(workflow_execution=ex).update(+ modified_at=created + timedelta(seconds=42)+ )++ self._call()++ ex.refresh_from_db()+ assert ex.status == ExecutionStatus.COMPLETED.value+ assert ex.execution_time == 42.0++ def test_execution_time_left_alone_when_no_file_timestamp(self):+ """No usable timestamp → keep whatever is there rather than write a worse+ guess. Pins the early return, which a refactor could silently drop.++ Exercised directly rather than through the endpoint: ``modified_at`` is+ ``auto_now`` and never NULL in practice, and a row with no file timestamp can+ no longer be SELECTED anyway (staleness is measured on the files now, and NULL+ does not satisfy ``< cutoff``). Driving it through the endpoint would assert+ the selection filter, not the early return this test exists to pin.+ """+ ex = self._exec(ExecutionStatus.EXECUTING, files=[ExecutionStatus.COMPLETED])+ # modified_at is NOT NULL at the database level, so it cannot be forced to NULL+ # to reach the early return — an earlier version of this test tried and died on+ # an IntegrityError. Patch the aggregate instead, which is the only way the+ # helper can legitimately see a missing timestamp.+ ex.execution_time = 7.5+ ex.save(update_fields=["execution_time"])++ with patch(+ "workflow_manager.file_execution.models.WorkflowFileExecution.objects"+ ) as objects:+ objects.filter.return_value.aggregate.return_value = {"last": None}+ self.view._restamp_execution_time_from_files(ex)++ ex.refresh_from_db()+ assert ex.execution_time == 7.5++ def test_execution_whose_files_JUST_finished_is_not_recovered(self):+ """The callback's window. ``workflow_execution.modified_at`` is effectively the+ START time — file completions write the file row, not the execution row — so any+ run longer than ``stuck_seconds`` is formally "stuck" while still running.++ The all-files-terminal filter stops that being catastrophic, but on its own it+ leaves a live race: between the last file going terminal and the callback+ finalizing, a perfectly healthy execution passes every other check. A sweep+ landing in that window finalizes it first, the terminal-one-way guard then+ refuses the callback's write, and the notification is silently lost.++ Measuring staleness on the files closes it. Mutation check: drop the+ ``last_file_at__lt=cutoff`` filter and this fails.+ """+ ex = self._exec(ExecutionStatus.EXECUTING, files=[ExecutionStatus.COMPLETED])+ _age(ex, 9000) # execution row looks ancient (it is just the start time)+ WorkflowFileExecution.objects.filter(workflow_execution=ex).update(+ modified_at=timezone.now() # ...but the work finished a moment ago+ )++ out = self._call(stuck_seconds=600)++ ex.refresh_from_db()+ assert out["scanned"] == 0+ assert ex.status == ExecutionStatus.EXECUTING.value++ def test_execution_whose_files_went_quiet_long_ago_IS_recovered(self):+ """The other half of the same predicate — moving staleness onto the files must+ not stop genuinely abandoned work from being recovered.+ """+ ex = self._exec(ExecutionStatus.EXECUTING, files=[ExecutionStatus.COMPLETED])+ _age(ex, 9000)+ WorkflowFileExecution.objects.filter(workflow_execution=ex).update(+ modified_at=timezone.now() - timedelta(seconds=3600)+ )++ out = self._call(stuck_seconds=600)+ ex.refresh_from_db()+ assert out["recovered"] == 1+ assert ex.status == ExecutionStatus.COMPLETED.valueclass TerminalOneWayGuardTests(TestCase):@@ -420,6 +638,8 @@ class ModelStaleWriterGuardTests(TestCase):assert ex.status == ExecutionStatus.COMPLETED.valueassert ex.successful_files == 1assert ex.result_acknowledged is True++class RetrieveNotFoundTests(TestCase):"""A missing execution must return 404, not 500 (UN-3719). The reaper'sorphan-claim sweep relies on the deterministic 404 to GC claims for deleteddiff --git a/backend/workflow_manager/internal_views.py b/backend/workflow_manager/internal_views.pyindex 4f9de9f0a..0aa0fb1f3 100644--- a/backend/workflow_manager/internal_views.py+++ b/backend/workflow_manager/internal_views.py@@ -646,12 +646,30 @@ class WorkflowExecutionInternalViewSet(viewsets.ReadOnlyModelViewSet):stuck past ``stuck_seconds`` whose files are ALL terminal, recomputes thecorrect terminal status from those files, and finalizes them.- Scoped to PG by ``queue_message_id`` (the durable per-row transport marker;- Celery rows carry ``task_id`` and ``queue_message_id=NULL``), so it needs no- Flipt context and never touches Celery executions.+ **Covers BOTH transports (UN-3796).** It was scoped to PG by+ ``queue_message_id__isnull=False``, which made a Celery execution invisible —+ so one stranded by a PG cutover (its workers removed while its chord callback+ was still to fire) sat ``EXECUTING`` forever with nothing able to close it. The+ finalization logic itself was always transport-agnostic: it reads file statuses+ and recomputes the terminal status, never touching a queue. Only the selection+ was narrow.++ Now requires the row to have been dispatched on **some** transport —+ ``queue_message_id`` (PG) or ``task_id`` (Celery). That deliberately excludes+ rows with BOTH handles NULL, which are never-dispatched and belong to+ ``workflow_v2/undispatched_sweep.py``; the two predicates are disjoint on+ ``task_id`` so they can never contend for the same row. (The+ ``total == 0 → skipped`` guard below would also catch those, but relying on a+ downstream guard for correctness is how overlaps get reintroduced.)++ The method and route keep the ``_pg_`` name despite no longer being PG-only:+ the URL is an internal-API contract between backend and workers, and renaming+ it would break during a rolling deploy where an older worker still calls the+ old path. Misleading name, deliberate trade."""from datetime import timedelta+ from django.db.models import Exists, Max, OuterRef, Qfrom django.utils import timezonefrom workflow_manager.workflow_v2.enums import ExecutionStatus@@ -672,15 +690,72 @@ class WorkflowExecutionInternalViewSet(viewsets.ReadOnlyModelViewSet):cutoff = timezone.now() - timedelta(seconds=stuck_seconds)terminal = ExecutionStatus.terminal_values()+ # Select ONLY rows the per-row guard below can actually finalize: at least one+ # file execution, and none of them still non-terminal.+ #+ # Without this the window fills with rows _recover_one_stuck_pg_execution()+ # always skips — and a skip leaves ``modified_at`` untouched, so the SAME rows+ # are re-selected on every sweep, forever. A genuinely recoverable execution+ # behind them is never reached: not scanned, not skipped, simply invisible.+ # Measured on integration during the UN-3796 cutover rehearsal: 1964 candidates,+ # of which 476 had no file rows and 646 had a non-terminal file — 57% permanent+ # skip fodder — while the freshly stranded execution sat at rank 1964 and every+ # sweep logged a healthy-looking "scanned 100, recovered 0".+ #+ # Filtering here rather than leaning on the downstream guard is also what makes+ # the drain real: every selected row finalizes and drops out of the candidate+ # set, so the backlog shrinks monotonically. That is precisely what keeps the+ # oldest-first ordering below starvation-free — FIFO is only fair if the queue+ # actually moves.+ files = WorkflowFileExecution.objects.filter(workflow_execution=OuterRef("pk"))stuck_ids = list(WorkflowExecution.objects.filter(- queue_message_id__isnull=False, # PG-only; Celery uses task_id+ # Dispatched by ANY evidence: the positive stamp, or either transport+ # handle. dispatched_at alone is not enough — pre-0027 rows never got+ # one — and the handles alone are not enough either, which is the gap+ # that makes this three-way rather than two.+ #+ # Once undispatched_sweep also requires `dispatched_at IS NULL`, a row+ # that WAS dispatched but whose handle write failed stops matching that+ # sweep. Without dispatched_at here it would match neither, and an+ # execution whose files all finished could sit non-terminal forever with+ # nothing able to close it — trading "wrongly terminalised" for "never+ # terminalised". The three-way test keeps the two sweeps disjoint AND+ # jointly exhaustive: undispatched takes rows where all three are absent,+ # this takes rows where any one is present.+ Q(dispatched_at__isnull=False)+ | Q(queue_message_id__isnull=False)+ | Q(task_id__isnull=False),status__in=[ExecutionStatus.PENDING.value,ExecutionStatus.EXECUTING.value,],modified_at__lt=cutoff,)+ .filter(Exists(files), ~Exists(files.exclude(status__in=terminal)))+ # Staleness must be measured on the FILES, not on the execution row.+ #+ # ``workflow_execution.modified_at`` is effectively the START time: file+ # completions write the file row, not the execution row, so it does not+ # advance while work proceeds. ``modified_at < cutoff`` therefore reads as+ # "started more than stuck_seconds ago", not "quiet for stuck_seconds" —+ # and every run longer than the window is formally eligible for recovery+ # WHILE IT IS STILL RUNNING. (Observed: a 14-minute ETL whose execution row+ # still read its 12:01:44 start at 12:15, with a 600 s window.)+ #+ # The all-files-terminal filter above stops that being catastrophic, but it+ # leaves a live race: between the last file going terminal and the callback+ # finalizing, a healthy execution passes every check here. If a sweep lands+ # in that window the reaper finalizes first, the terminal-one-way guard then+ # refuses the callback's own write, and the notification never fires.+ #+ # Requiring the LAST FILE to also be older than the cutoff closes it: an+ # execution whose files just finished is not yet stale, so the callback keeps+ # its window and only genuinely abandoned work is recovered. Both conditions+ # are kept — the indexed execution-row predicate cheaply narrows the scan,+ # this one decides correctness.+ .annotate(last_file_at=Max("file_executions__modified_at"))+ .filter(last_file_at__lt=cutoff).order_by("modified_at").values_list("id", flat=True)[:limit])@@ -753,6 +828,7 @@ class WorkflowExecutionInternalViewSet(viewsets.ReadOnlyModelViewSet):else:computed = ExecutionStatus.COMPLETEDexecution.update_execution(status=computed)+ self._restamp_execution_time_from_files(execution)self._update_file_aggregates(execution,{@@ -828,6 +904,35 @@ class WorkflowExecutionInternalViewSet(viewsets.ReadOnlyModelViewSet):return error_msg[:253] + "..."return error_msg+ @staticmethod+ def _restamp_execution_time_from_files(execution) -> None:+ """Correct the runtime ``update_execution()`` just stamped from ``now()``.++ Both transports' status writers set ``execution_time = now - created_at`` on+ any terminal transition (``_apply_legacy_update`` / ``_apply_pg_guarded_update``+ in the execution model). That is right for a live finalization, where "now" IS+ when the work ended, and wrong here: this sweep finalizes executions whose files+ finished long before anything noticed, so the stamp would record how late the+ reaper was rather than how long the work took. On the integration backlog that+ meant multi-day runtimes on executions that ran for minutes — every one of them+ feeding whatever dashboards read this column.++ Recompute from the last file to reach a terminal state, which is the closest+ durable record of when the work actually ended. If no file carries a timestamp,+ leave whatever is there alone rather than substituting a worse guess.+ """+ from django.db.models import Max++ last_file_at = WorkflowFileExecution.objects.filter(+ workflow_execution=execution+ ).aggregate(last=Max("modified_at"))["last"]+ if not last_file_at or not execution.created_at:+ return+ execution.execution_time = round(+ (last_file_at - execution.created_at).total_seconds(), 3+ )+ execution.save(update_fields=["execution_time"])+@staticmethoddef _update_file_aggregates(execution, validated_data) -> None:"""Persist total / successful / failed file counts when present."""diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex cbe02d278..213c81302 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -409,6 +409,37 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):except WorkflowExecution.DoesNotExist:logger.error(f"execution doesn't exist {execution_id}")+ @staticmethod+ def mark_dispatched(execution_id: str) -> None:+ """Stamp ``dispatched_at`` — the positive record that the orchestrator task+ reached its transport.++ Called by the dispatcher the instant dispatch returns, BEFORE any handle+ bookkeeping. That ordering is the whole point: ``_record_dispatch_handle`` has+ three paths that leave both handles NULL on an execution that IS running (an+ exception the caller swallows, an empty handle, an unparseable PG handle), and+ the undispatched sweep used to read that absence as "never dispatched" and mark+ the live row ERROR. One write upstream of all three closes it.++ Best-effort, matching the handle write it precedes: a failure here must never+ abort the caller, which is past the point of no return — the orchestrator is+ already running. A failure leaves exactly the pre-existing ambiguity rather than+ a new one.++ ``update()`` rather than ``save()``: no auto_now field on this model may be+ re-stamped by a bookkeeping write, and this must not race the orchestrator's own+ status writes on the same row.+ """+ from django.utils import timezone++ updated = WorkflowExecution.objects.filter(id=execution_id).update(+ dispatched_at=timezone.now()+ )+ if not updated:+ logger.error(+ f"Could not stamp dispatched_at: execution {execution_id} not found"+ )+@staticmethoddef update_execution_queue_message_id(execution_id: str, queue_message_id: int | Nonediff --git a/backend/workflow_manager/workflow_v2/execution_log_internal_urls.py b/backend/workflow_manager/workflow_v2/execution_log_internal_urls.pyindex 7f60dc88d..d59d85268 100644--- a/backend/workflow_manager/workflow_v2/execution_log_internal_urls.py+++ b/backend/workflow_manager/workflow_v2/execution_log_internal_urls.py@@ -33,4 +33,9 @@ urlpatterns = [execution_log_internal_views.ProcessLogHistoryAPIView.as_view(),name="process_log_history",),+ path(+ "sweep-undispatched-executions/",+ execution_log_internal_views.SweepUndispatchedExecutionsAPIView.as_view(),+ name="sweep_undispatched_executions",+ ),]diff --git a/backend/workflow_manager/workflow_v2/execution_log_internal_views.py b/backend/workflow_manager/workflow_v2/execution_log_internal_views.pyindex 708376dcc..0c07b0474 100644--- a/backend/workflow_manager/workflow_v2/execution_log_internal_views.py+++ b/backend/workflow_manager/workflow_v2/execution_log_internal_views.py@@ -16,6 +16,9 @@ from workflow_manager.workflow_v2.execution_log_utils import (process_log_history_from_cache,)from workflow_manager.workflow_v2.models import WorkflowExecution+from workflow_manager.workflow_v2.undispatched_sweep import (+ sweep_undispatched_executions,+)logger = logging.getLogger(__name__)@@ -171,3 +174,32 @@ class ProcessLogHistoryAPIView(APIView):return Response({"error": str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)+++class SweepUndispatchedExecutionsAPIView(APIView):+ """Terminalise executions that were created but never dispatched.++ Called by the PG-queue reaper on its sweep tick. The reaper owns *when* — it is+ leader-elected, so exactly one instance sweeps — and the backend owns the *logic*,+ mirroring ProcessLogHistoryAPIView above.++ It lives here rather than in the reaper because the reaper deliberately never+ touches backend tables: it reads execution state through+ ``get_workflow_execution`` and writes through ``update_workflow_execution_status``.+ A ``WorkflowExecution`` query inside the worker would break that boundary, and the+ sweep also needs the rate limiter — both backend-side.++ Transport-agnostic on purpose: the create-then-dispatch window sits upstream of+ ``resolve_transport``, so this recovers Celery-path strands too.+ """++ def post(self, request: Request) -> Response:+ """Run one sweep. Returns the number of executions terminalised."""+ try:+ swept = sweep_undispatched_executions()+ return Response({"swept": swept})+ except Exception as e:+ logger.error(f"Error sweeping undispatched executions: {e}", exc_info=True)+ return Response(+ {"error": str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR+ )diff --git a/backend/workflow_manager/workflow_v2/migrations/0026_workflowexecution_undispatched_idx.py b/backend/workflow_manager/workflow_v2/migrations/0026_workflowexecution_undispatched_idx.pynew file mode 100644index 000000000..c8b7157d4--- /dev/null+++ b/backend/workflow_manager/workflow_v2/migrations/0026_workflowexecution_undispatched_idx.py@@ -0,0 +1,132 @@+"""Add a partial index over UNDISPATCHED workflow executions.++Serves the undispatched-execution sweep (``workflow_v2/undispatched_sweep.py``), which+terminalises rows left ``PENDING`` because the request died between+``create_workflow_execution`` and ``execute_workflow_async`` — committed but never+queued, so no ``pg_barrier_state`` row exists and the reaper's barrier scan cannot see+them. Measured on integration 2026-08-17: 967 such rows from a single 5-minute load+test whose API tier shed 71% of its traffic.++The sweep runs every 5 minutes (``WORKER_PG_REAPER_SWEEP_SECONDS``) with::++ WHERE status = 'PENDING' AND created_at < now() - grace+ AND task_id IS NULL AND queue_message_id IS NULL++Design+------+* PARTIAL — the index covers only rows that are simultaneously PENDING and+ undispatched. In steady state that is ~ZERO rows: an execution leaves PENDING within+ seconds of creation, and one of ``task_id`` / ``queue_message_id`` is stamped the+ moment dispatch succeeds. So the index is effectively empty, costs almost nothing to+ maintain, and only grows when something is actually wrong.+* ``created_at`` as the indexed column — the sweep's only range predicate, and the+ ORDER BY for the batch limit.+* WHY A NEW INDEX AT ALL. ``we_active_by_workflow_idx`` (migration 0023) *is* usable+ here — ``status = 'PENDING'`` provably implies its ``NOT IN (terminal)`` predicate —+ so without this the fallback is a full scan of that (small) partial index rather than+ a table seq scan. That is survivable but planner-dependent and grows with the count+ of active executions. This makes it an exact index scan over a near-empty index.+* CONCURRENTLY + ``atomic = False`` — ``workflow_execution`` is a multi-million-row+ table in production; a plain ``AddIndex`` holds a SHARE lock for the whole build and+ blocks writes, i.e. blocks every execution in flight.+* FROZEN LITERAL — migrations must not import app enums, so ``'PENDING'`` is hardcoded.+ Drift against ``ExecutionStatus.PENDING`` is caught by+ ``tests/test_undispatched_execution_index.py`` (model/enum introspection only, no+ test DB), mirroring what ``test_active_execution_index.py`` does for 0023.+* INVALID-INDEX GUARD — ``IF NOT EXISTS`` silently no-ops over a leftover INVALID index+ from an interrupted CONCURRENTLY build, and Django would then record this migration as+ applied while the index is physically unusable (never read, write overhead only). The+ second statement RAISEs in that case, so the failure is loud rather than green-but-broken.++Deployment+----------+``CREATE INDEX CONCURRENTLY`` scans the table and can run for minutes on a large table —+long enough to time out a deploy's ``migrate`` step. Prefer building it OUT OF BAND+*before* the deploy; the migration then no-ops via ``IF NOT EXISTS``::++ CREATE INDEX CONCURRENTLY IF NOT EXISTS we_undispatched_idx+ ON workflow_execution (created_at)+ WHERE status = 'PENDING'+ AND task_id IS NULL+ AND queue_message_id IS NULL;++Then confirm the planner uses it (the partial-index proof depends on the literals+reaching the planner as constants)::++ EXPLAIN SELECT id, workflow_id FROM workflow_execution+ WHERE status = 'PENDING' AND created_at < now() - interval '15 minutes'+ AND task_id IS NULL AND queue_message_id IS NULL+ ORDER BY created_at LIMIT 500;+ -- expect: Index Scan using we_undispatched_idx++Recovery+--------+An interrupted build leaves an INVALID index that adds write overhead but is never read.+``IF NOT EXISTS`` will NOT rebuild over it (and the guard below RAISEs on it), so drop it+first and re-run::++ DROP INDEX CONCURRENTLY IF EXISTS we_undispatched_idx;+"""++from django.db import migrations, models+from django.db.models import Q++INDEX_NAME = "we_undispatched_idx"++# Frozen — see the FROZEN LITERAL note above.+PENDING_STATUS = "PENDING"++_ASSERT_INDEX_VALID = f"""+DO $$+BEGIN+ IF EXISTS (+ SELECT 1 FROM pg_class c+ JOIN pg_index i ON i.indexrelid = c.oid+ WHERE c.relname = '{INDEX_NAME}' AND NOT i.indisvalid+ ) THEN+ RAISE EXCEPTION 'Index {INDEX_NAME} exists but is INVALID (a prior CREATE INDEX CONCURRENTLY was interrupted). Drop it and re-run: DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME};';+ END IF;+END+$$;+"""+++class Migration(migrations.Migration):+ # CREATE / DROP INDEX CONCURRENTLY cannot run inside a transaction block.+ atomic = False++ dependencies = [("workflow_v2", "0025_workflow_workflow_org_modified_idx")]++ operations = [+ migrations.SeparateDatabaseAndState(+ database_operations=[+ migrations.RunSQL(+ sql=(+ f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {INDEX_NAME} "+ "ON workflow_execution (created_at) "+ f"WHERE status = '{PENDING_STATUS}' "+ "AND task_id IS NULL "+ "AND queue_message_id IS NULL;"+ ),+ reverse_sql=f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME};",+ ),+ migrations.RunSQL(+ sql=_ASSERT_INDEX_VALID, reverse_sql=migrations.RunSQL.noop+ ),+ ],+ state_operations=[+ migrations.AddIndex(+ model_name="workflowexecution",+ index=models.Index(+ fields=["created_at"],+ name=INDEX_NAME,+ condition=Q(+ status=PENDING_STATUS,+ task_id__isnull=True,+ queue_message_id__isnull=True,+ ),+ ),+ ),+ ],+ ),+ ]diff --git a/backend/workflow_manager/workflow_v2/migrations/0027_workflowexecution_dispatched_at.py b/backend/workflow_manager/workflow_v2/migrations/0027_workflowexecution_dispatched_at.pynew file mode 100644index 000000000..9d483d152--- /dev/null+++ b/backend/workflow_manager/workflow_v2/migrations/0027_workflowexecution_dispatched_at.py@@ -0,0 +1,69 @@+"""Record dispatch as a POSITIVE fact instead of inferring it from absent handles.++The undispatched sweep (``workflow_v2/undispatched_sweep.py``) decides that an+execution "never dispatched" from ``task_id IS NULL AND queue_message_id IS NULL``.+That is an inference, and it is unsound: three paths in ``workflow_helper`` reach+exactly that state AFTER the message is already on its transport —++* ``_record_dispatch_handle`` raising and the caller swallowing it, under the comment+ "continuing — the orchestrator is already running";+* the handle coming back empty and the recorder returning without writing;+* a PG handle that will not parse as a bigint, same early return.++So the sweep could claim a RUNNING execution, mark it ERROR and tell the user "You can+safely run it again". On the Celery transport that self-corrected — the orchestrator ran+regardless and its terminal write superseded the ERROR. On PG it does NOT: both worker+entry points stop on a terminal execution (``general/tasks.py`` returns+``skipped_terminal_execution``; ``file_processing/tasks.py`` raises+``_TerminalExecutionSkip``), so the message is acked and the work is silently DROPPED.++``dispatched_at`` removes the inference. It is stamped by the dispatcher the moment+dispatch succeeds, BEFORE any handle bookkeeping can fail, so all three paths above are+covered by one write.++Nullable and additive+---------------------+No backfill, and none is needed. Every pre-existing row has ``dispatched_at IS NULL``+whatever its true state, so the sweep predicate KEEPS the two handle checks alongside the+new one::++ WHERE status = 'PENDING' AND created_at < now() - grace+ AND dispatched_at IS NULL+ AND task_id IS NULL AND queue_message_id IS NULL++That is what makes this a ONE-PHASE deploy. During a rolling upgrade an old backend pod+dispatches without stamping; the row then matches ``dispatched_at IS NULL`` but is+excluded by ``task_id IS NOT NULL``. Drop the handle checks and you would need to stamp+everywhere first and switch the predicate in a later release, or sweep live work+mid-deploy.++The index over that predicate is swapped separately in 0028, which needs+``atomic = False`` for ``CREATE INDEX CONCURRENTLY``; keeping the column addition here+means this migration stays a fast, transactional, single-purpose change.+"""++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("workflow_v2", "0026_workflowexecution_undispatched_idx"),+ ]++ operations = [+ migrations.AddField(+ model_name="workflowexecution",+ name="dispatched_at",+ field=models.DateTimeField(+ null=True,+ blank=True,+ db_comment=(+ "Set when the orchestrator task was handed to its transport. NULL means "+ "never dispatched. The POSITIVE fact the undispatched sweep needs: absent "+ "task_id/queue_message_id also occurs for executions that ARE running, "+ "because three paths in workflow_helper return without recording a handle "+ "after the message is already enqueued."+ ),+ ),+ ),+ ]diff --git a/backend/workflow_manager/workflow_v2/migrations/0028_undispatched_idx_dispatched_at.py b/backend/workflow_manager/workflow_v2/migrations/0028_undispatched_idx_dispatched_at.pynew file mode 100644index 000000000..e6cd88710--- /dev/null+++ b/backend/workflow_manager/workflow_v2/migrations/0028_undispatched_idx_dispatched_at.py@@ -0,0 +1,136 @@+"""Re-key the undispatched partial index on ``dispatched_at``.++0027 added the column and explains why the inference it replaces was unsound. This+swaps the index that serves the sweep so the new predicate stays an index scan rather+than degrading to a sequential scan of a multi-million-row table.++The sweep now runs::++ WHERE status = 'PENDING' AND created_at < now() - grace+ AND dispatched_at IS NULL+ AND task_id IS NULL AND queue_message_id IS NULL++The handle checks are KEPT deliberately — see 0027 for why they are what make the+rollout single-phase — so the index condition carries all three.++Ordering matters here. The NEW index is created before the OLD one is dropped, so the+predicate is served throughout: a deploy that is interrupted between the two operations+leaves both present, which costs a little write overhead and nothing else. Dropping+first would leave the sweep unindexed for the length of the build.++Same operational rules as 0026, which this mirrors:++* ``atomic = False`` — CREATE / DROP INDEX CONCURRENTLY cannot run in a transaction.+* Prefer building OUT OF BAND before the deploy; ``IF NOT EXISTS`` then no-ops::++ CREATE INDEX CONCURRENTLY IF NOT EXISTS we_undispatched_dispatch_idx+ ON workflow_execution (created_at)+ WHERE status = 'PENDING'+ AND dispatched_at IS NULL+ AND task_id IS NULL+ AND queue_message_id IS NULL;++* An interrupted build leaves an INVALID index that is never read but still costs+ writes. ``IF NOT EXISTS`` will not rebuild over it and the guard below RAISEs on it,+ so drop and re-run::++ DROP INDEX CONCURRENTLY IF EXISTS we_undispatched_dispatch_idx;++* Confirm the planner uses it — the partial-index proof depends on the literals+ reaching the planner as constants::++ EXPLAIN SELECT id, workflow_id FROM workflow_execution+ WHERE status = 'PENDING' AND created_at < now() - interval '1 hour'+ AND dispatched_at IS NULL+ AND task_id IS NULL AND queue_message_id IS NULL+ ORDER BY created_at LIMIT 500;+ -- expect: Index Scan using we_undispatched_dispatch_idx++A NEW NAME rather than a rebuild under the old one: an index cannot be redefined in+place, and reusing ``we_undispatched_idx`` would mean dropping before creating, leaving+the window this ordering exists to avoid.+"""++from django.db import migrations, models+from django.db.models import Q++OLD_INDEX_NAME = "we_undispatched_idx"+INDEX_NAME = "we_undispatched_dispatch_idx"++# Frozen literal, tied to ExecutionStatus by+# tests/test_undispatched_execution_index.py — see 0026.+PENDING_STATUS = "PENDING"++_ASSERT_INDEX_VALID = f"""+DO $$+BEGIN+ IF EXISTS (+ SELECT 1 FROM pg_class c+ JOIN pg_index i ON i.indexrelid = c.oid+ WHERE c.relname = '{INDEX_NAME}' AND NOT i.indisvalid+ ) THEN+ RAISE EXCEPTION 'Index {INDEX_NAME} exists but is INVALID (a prior CREATE INDEX CONCURRENTLY was interrupted). Drop it and re-run: DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME};';+ END IF;+END+$$;+"""+++class Migration(migrations.Migration):+ # CREATE / DROP INDEX CONCURRENTLY cannot run inside a transaction block.+ atomic = False++ dependencies = [("workflow_v2", "0027_workflowexecution_dispatched_at")]++ operations = [+ migrations.SeparateDatabaseAndState(+ database_operations=[+ # Create the replacement FIRST — the predicate stays served throughout.+ migrations.RunSQL(+ sql=(+ f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {INDEX_NAME} "+ "ON workflow_execution (created_at) "+ f"WHERE status = '{PENDING_STATUS}' "+ "AND dispatched_at IS NULL "+ "AND task_id IS NULL "+ "AND queue_message_id IS NULL;"+ ),+ reverse_sql=f"DROP INDEX CONCURRENTLY IF EXISTS {INDEX_NAME};",+ ),+ migrations.RunSQL(+ sql=_ASSERT_INDEX_VALID, reverse_sql=migrations.RunSQL.noop+ ),+ # Only then retire the old one. Reverse recreates it, so a rollback of+ # this migration leaves the pre-0028 predicate indexed.+ migrations.RunSQL(+ sql=f"DROP INDEX CONCURRENTLY IF EXISTS {OLD_INDEX_NAME};",+ reverse_sql=(+ f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {OLD_INDEX_NAME} "+ "ON workflow_execution (created_at) "+ f"WHERE status = '{PENDING_STATUS}' "+ "AND task_id IS NULL "+ "AND queue_message_id IS NULL;"+ ),+ ),+ ],+ state_operations=[+ migrations.RemoveIndex(+ model_name="workflowexecution",+ name=OLD_INDEX_NAME,+ ),+ migrations.AddIndex(+ model_name="workflowexecution",+ index=models.Index(+ fields=["created_at"],+ name=INDEX_NAME,+ condition=Q(+ status=PENDING_STATUS,+ dispatched_at__isnull=True,+ task_id__isnull=True,+ queue_message_id__isnull=True,+ ),+ ),+ ),+ ],+ ),+ ]diff --git a/backend/workflow_manager/workflow_v2/models/execution.py b/backend/workflow_manager/workflow_v2/models/execution.pyindex a0ec40ae6..d6082aa42 100644--- a/backend/workflow_manager/workflow_v2/models/execution.py+++ b/backend/workflow_manager/workflow_v2/models/execution.py@@ -213,6 +213,17 @@ class WorkflowExecution(BaseModel):db_comment="Details of encountered errors",)attempts = models.IntegerField(default=0, db_comment="number of attempts taken")+ dispatched_at = models.DateTimeField(+ null=True,+ blank=True,+ db_comment=(+ "Set when the orchestrator task was handed to its transport. NULL means "+ "never dispatched. The POSITIVE fact the undispatched sweep needs: absent "+ "task_id/queue_message_id also occurs for executions that ARE running, "+ "because three paths in workflow_helper return without recording a handle "+ "after the message is already enqueued."+ ),+ )execution_time = models.FloatField(default=0, db_comment="execution time in seconds")tags = models.ManyToManyField(Tag, related_name="workflow_executions", blank=True)@@ -239,6 +250,28 @@ class WorkflowExecution(BaseModel):name="we_active_by_workflow_idx",condition=~Q(status__in=["COMPLETED", "STOPPED", "ERROR"]),),+ # Partial index over UNDISPATCHED executions — see migration 0026.+ # Serves the undispatched-execution sweep (undispatched_sweep.py), which+ # terminalises rows left PENDING because the request died between+ # create_workflow_execution and execute_workflow_async. Effectively EMPTY+ # in steady state: an execution leaves PENDING within seconds, and one of+ # task_id / queue_message_id is stamped the moment dispatch succeeds — so+ # it costs almost nothing and only grows when something is wrong.+ # we_active_by_workflow_idx above is *usable* for the same predicate+ # (PENDING implies NOT IN terminal) but is keyed on workflow_id, which the+ # sweep does not filter on — so without this the sweep falls back to a full+ # scan of that index. Literals are frozen in the migration and tied to+ # ExecutionStatus by tests/test_undispatched_execution_index.py.+ models.Index(+ fields=["created_at"],+ name="we_undispatched_dispatch_idx",+ condition=Q(+ status="PENDING",+ dispatched_at__isnull=True,+ task_id__isnull=True,+ queue_message_id__isnull=True,+ ),+ ),]@propertydiff --git a/backend/workflow_manager/workflow_v2/tests/test_step_execution_transport.py b/backend/workflow_manager/workflow_v2/tests/test_step_execution_transport.pynew file mode 100644index 000000000..2e179ad0e--- /dev/null+++ b/backend/workflow_manager/workflow_v2/tests/test_step_execution_transport.py@@ -0,0 +1,120 @@+"""Tests for the PG guard on single-step execution (UN-3445 acceptance gate).++Single-step is the one entry path whose fan-out was never transport-gated: it+reaches the Celery chord in ``process_input_files`` directly, while the normal+path's PG fan-out lives in the general worker. With the flag on and the Celery+file_processing workers scaled to zero, those batches would never be consumed+and the execution would hang in EXECUTING with no PG reaper aware of it.++So the guard has to hold in *both* directions, and that is what these pin:+Celery must still run step execution unchanged (it is what staging and+production run today), and PG must refuse it loudly rather than dispatch into a+queue with no consumer.++DB-free: WorkflowExecution, resolve_transport and run_workflow are all mocked.+"""++from unittest.mock import MagicMock, patch++import pytest+from workflow_manager.workflow_v2.exceptions import WorkflowExecutionError+from workflow_manager.workflow_v2.models import Workflow+from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper++_TRANSPORT = "workflow_manager.workflow_v2.workflow_helper.resolve_transport"+_EXECUTION = "workflow_manager.workflow_v2.workflow_helper.WorkflowExecution"+_RUN = "workflow_manager.workflow_v2.workflow_helper.WorkflowHelper.run_workflow"++# The production comparison is `execution_action is ...START.value`, so the test+# must pass that exact object rather than an equal string literal.+_START = Workflow.ExecutionAction.START.value+_EXECUTION_ID = "11111111-1111-1111-1111-111111111111"+++def _workflow():+ workflow = MagicMock()+ workflow.organization.organization_id = "org_acme"+ return workflow+++def _patched_execution():+ """Mock WorkflowExecution, keeping DoesNotExist a real exception class.++ A bare MagicMock attribute is not catchable, so the production+ ``except WorkflowExecution.DoesNotExist`` would raise TypeError instead.+ """+ model = MagicMock()+ model.DoesNotExist = type("DoesNotExist", (Exception,), {})+ model.objects.get.return_value = MagicMock()+ return model+++class TestStepExecutionTransportGuard:+ def test_pg_transport_refuses_instead_of_dispatching_to_celery(self):+ with patch(_EXECUTION, _patched_execution()), patch(+ _TRANSPORT, return_value="pg_queue"+ ), patch(_RUN) as run:+ with pytest.raises(WorkflowExecutionError, match="not supported"):+ WorkflowHelper.step_execution(+ workflow=_workflow(),+ execution_action=_START,+ execution_id=_EXECUTION_ID,+ )+ # The point of the guard: nothing reaches the chord. Asserting the raise+ # alone would still pass if run_workflow had already fanned out.+ run.assert_not_called()++ def test_celery_transport_still_runs_step_execution(self):+ """Flag-off is what staging and production run — it must be untouched."""+ with patch(_EXECUTION, _patched_execution()), patch(+ _TRANSPORT, return_value="celery"+ ), patch(_RUN, return_value="ran") as run:+ result = WorkflowHelper.step_execution(+ workflow=_workflow(),+ execution_action=_START,+ execution_id=_EXECUTION_ID,+ )+ assert result == "ran"+ assert run.call_args.kwargs["single_step"] is True++ def test_transport_is_resolved_on_the_execution_id(self):+ """entity_id must be the execution id — it is what Flipt buckets on, and+ what keeps one execution from re-bucketing across transports."""+ with patch(_EXECUTION, _patched_execution()), patch(+ _TRANSPORT, return_value="celery"+ ) as resolve, patch(_RUN):+ WorkflowHelper.step_execution(+ workflow=_workflow(),+ execution_action=_START,+ execution_id=_EXECUTION_ID,+ )+ assert resolve.call_args.kwargs["execution_id"] == _EXECUTION_ID+ assert resolve.call_args.kwargs["organization_id"] == "org_acme"++ def test_no_execution_id_creates_a_step_execution_without_resolving(self):+ """The START-without-id branch only mints a row; it dispatches nothing, so+ it must not consume a Flipt evaluation (and must not be blocked)."""+ with patch(_TRANSPORT) as resolve, patch.object(+ WorkflowHelper, "create_and_make_execution_response", return_value="created"+ ):+ result = WorkflowHelper.step_execution(+ workflow=_workflow(), execution_action=_START, execution_id=None+ )+ assert result == "created"+ resolve.assert_not_called()++ def test_missing_execution_falls_back_to_creating_one(self):+ """Pre-existing behaviour: a stale execution_id re-mints rather than 404s.+ Pinned so the guard's placement inside the try/except cannot change it."""+ model = _patched_execution()+ model.objects.get.side_effect = model.DoesNotExist+ with patch(_EXECUTION, model), patch(_TRANSPORT) as resolve, patch.object(+ WorkflowHelper, "create_and_make_execution_response", return_value="created"+ ):+ result = WorkflowHelper.step_execution(+ workflow=_workflow(),+ execution_action=_START,+ execution_id=_EXECUTION_ID,+ )+ assert result == "created"+ resolve.assert_not_called()diff --git a/backend/workflow_manager/workflow_v2/tests/test_undispatched_execution_index.py b/backend/workflow_manager/workflow_v2/tests/test_undispatched_execution_index.pynew file mode 100644index 000000000..4268a2f7f--- /dev/null+++ b/backend/workflow_manager/workflow_v2/tests/test_undispatched_execution_index.py@@ -0,0 +1,149 @@+"""Guard: the undispatched-execution partial index stays in sync with reality.++``we_undispatched_dispatch_idx`` (``WorkflowExecution.Meta.indexes``) hardcodes the literal+``'PENDING'`` and the two dispatch-handle columns. The same predicate is written a+second time in migration 0026's ``RunSQL``, and a third time as the sweep's WHERE clause+in ``undispatched_sweep.py`` — migrations cannot import app enums, so the literal cannot+simply reference ``ExecutionStatus``.++Three copies of one predicate is exactly how an index silently stops matching the query+it exists for: the sweep keeps working, just without the index, and nobody notices until+it is scanning a multi-million-row table every 5 minutes. These assert the copies agree.++Model introspection only — no test database required, so this runs in the unit tier+alongside ``test_active_execution_index.py``, which does the same job for 0023.+"""++from __future__ import annotations++import os+import re+from pathlib import Path++import django+from django.apps import apps++os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings.test")+if not apps.ready:+ django.setup()++from unstract.core.data_models import ExecutionStatus # noqa: E402++INDEX_NAME = "we_undispatched_dispatch_idx"+_MIGRATION = (+ Path(__file__).resolve().parent.parent+ / "migrations"+ / "0026_workflowexecution_undispatched_idx.py"+)+++def _index():+ model = apps.get_model("workflow_v2", "WorkflowExecution")+ return next((i for i in model._meta.indexes if i.name == INDEX_NAME), None)+++def _condition_children() -> dict:+ index = _index()+ assert index is not None, f"{INDEX_NAME} is missing from WorkflowExecution.Meta"+ return {c[0]: c[1] for c in index.condition.children if isinstance(c, tuple)}+++class TestTheModelIndexMatchesTheEnum:+ def test_the_status_literal_is_the_pending_enum_value(self):+ """If ExecutionStatus.PENDING's *value* ever changes, the index silently stops+ matching every row the sweep looks for."""+ assert _condition_children()["status"] == ExecutionStatus.PENDING.value++ def test_all_three_dispatch_signals_are_in_the_predicate(self):+ """Dropping any makes the index non-matching for the sweep's WHERE clause+ (Postgres needs the query predicate to imply the index predicate), so the sweep+ would quietly fall back to scanning.++ Three, not two: `dispatched_at` is the authoritative test (0027), and the two+ handle checks are retained so the rollout is single-phase — a row dispatched by+ an old pod mid-deploy has no stamp but does have a handle, and must not be swept.+ """+ children = _condition_children()+ assert children.get("dispatched_at__isnull") is True+ assert children.get("task_id__isnull") is True+ assert children.get("queue_message_id__isnull") is True++ def test_it_is_keyed_on_created_at(self):+ """The sweep's only range predicate and its ORDER BY for the batch limit."""+ assert _index().fields == ["created_at"]+++class TestTheMigrationMatchesTheModel:+ """The migration's raw SQL is the copy that actually builds the index; the model's+ Index() is only Django state. They must not disagree.+ """++ def test_the_migration_sql_uses_the_same_status_literal(self):+ sql = _MIGRATION.read_text()+ assert f"status = '{ExecutionStatus.PENDING.value}'" in sql++ def test_the_migration_sql_carries_both_handle_conditions(self):+ sql = _MIGRATION.read_text()+ assert "task_id IS NULL" in sql+ assert "queue_message_id IS NULL" in sql++ def test_the_migration_builds_concurrently_and_is_non_atomic(self):+ """workflow_execution is multi-million-row in production — a plain AddIndex+ holds a SHARE lock for the whole build and blocks every in-flight execution.+ """+ sql = _MIGRATION.read_text()+ assert "CREATE INDEX CONCURRENTLY IF NOT EXISTS" in sql+ assert re.search(r"^\s*atomic\s*=\s*False", sql, re.MULTILINE)++ def test_it_guards_against_a_leftover_invalid_index(self):+ """IF NOT EXISTS no-ops over an INVALID index from an interrupted build, and+ Django would record the migration as applied over something unusable.+ """+ sql = _MIGRATION.read_text()+ assert "RAISE EXCEPTION" in sql+ assert "indisvalid" in sql+++class TestTheSweepQueryMatchesTheIndex:+ def test_the_sweep_predicate_uses_the_same_columns(self):+ """The third copy. If the sweep's SQL drifts from the index predicate, Postgres+ can no longer prove the implication and the partial index becomes unusable —+ the sweep still returns correct rows, just by scanning.+ """+ from workflow_manager.workflow_v2 import undispatched_sweep++ sql = undispatched_sweep._CLAIM_SQL+ assert "task_id IS NULL" in sql+ assert "queue_message_id IS NULL" in sql+ assert "created_at <" in sql+ assert "ORDER BY created_at" in sql++ def test_the_predicate_appears_TWICE__inner_select_and_outer_recheck(self):+ """The race guard, and the one thing a substring check silently misses.++ The inner SELECT picks and locks candidates; the OUTER WHERE re-carries the same+ predicate so Postgres re-evaluates it at write time under those locks. Drop it+ from the outer clause and the statement still looks right — the strings are all+ still present, courtesy of the inner SELECT — but a row dispatched between the+ two is now marked ERROR *while it runs*.++ Verified: removing the outer re-check passed every other test in this file.+ """+ from workflow_manager.workflow_v2 import undispatched_sweep++ sql = undispatched_sweep._CLAIM_SQL+ for clause in ("task_id IS NULL", "queue_message_id IS NULL"):+ assert sql.count(clause) >= 2, (+ f"{clause!r} appears {sql.count(clause)}x — it must be in BOTH the "+ "inner SELECT and the outer re-check, or the claim is racy"+ )+ # The status check likewise: %s placeholders, so count the column instead.+ assert sql.count("status = %s") >= 2, "status must be re-checked in the UPDATE"++ def test_it_locks_candidates_with_skip_locked(self):+ """Without FOR UPDATE the outer re-check reads unlocked rows and the race+ reopens; without SKIP LOCKED an overlapping sweeper blocks instead of yielding.+ """+ from workflow_manager.workflow_v2 import undispatched_sweep++ assert "FOR UPDATE SKIP LOCKED" in undispatched_sweep._CLAIM_SQLdiff --git a/backend/workflow_manager/workflow_v2/tests/test_undispatched_sweep.py b/backend/workflow_manager/workflow_v2/tests/test_undispatched_sweep.pynew file mode 100644index 000000000..e69c2d42d--- /dev/null+++ b/backend/workflow_manager/workflow_v2/tests/test_undispatched_sweep.py@@ -0,0 +1,358 @@+"""PENDING is not terminal — an undispatched execution must not stay that way.++Guards the sweep that closes the create-then-dispatch window (see the module+docstring). The two failure modes that matter are opposite in direction, and both are+pinned here:++ * FAILING TO SWEEP leaves an execution that can never reach COMPLETED or ERROR —+ invisible failure. 967 of them appeared from one 5-minute load test.+ * SWEEPING TOO EAGERLY marks a LIVE execution ERROR. Far worse, so the young-row,+ dispatched-row and raced-row cases are asserted explicitly rather than assumed.+"""++from __future__ import annotations++import uuid+from datetime import timedelta++import pytest+from django.utils import timezone++from unstract.core.data_models import ExecutionStatus++# NOT module-level: TestUserFacingMessage asserts a pure string contract and needs+# no database, so it must stay in the UNIT tier. conftest auto-marks django_db+# tests as `integration`, which the unit lane excludes — a module-level marker+# would have hidden the message assertions behind live Postgres for no reason.+++def _execution(**overrides):+ """A PENDING, never-dispatched execution old enough to sweep.++ A workflow is REQUIRED, not incidental: ``WorkflowExecution.save()`` calls+ ``_handle_execution_cache()``, which dereferences ``self.workflow.id``. Built+ without one, every test here died with ``AttributeError: 'NoneType' object has no+ attribute 'id'`` before reaching its assertion — which is what CI reported the+ moment this PR left draft and the integration tier actually ran.+ """+ from workflow_manager.workflow_v2.models import Workflow, WorkflowExecution++ fields = {+ "id": uuid.uuid4(),+ "workflow": Workflow.objects.create(+ workflow_name=f"wf-sweep-{uuid.uuid4().hex[:8]}"+ ),+ "status": ExecutionStatus.PENDING.value,+ "task_id": None,+ "queue_message_id": None,+ }+ fields.update(overrides)+ created_at = fields.pop("created_at", timezone.now() - timedelta(hours=2))+ ex = WorkflowExecution.objects.create(**fields)+ # created_at is auto_now_add; bypass it with a direct UPDATE so age is controllable.+ WorkflowExecution.objects.filter(id=ex.id).update(created_at=created_at)+ ex.refresh_from_db()+ return ex+++class TestSweepsWhatNothingElseOwns:+ def test_an_aged_undispatched_execution_is_marked_error(self):+ from workflow_manager.workflow_v2.models import WorkflowExecution+ from workflow_manager.workflow_v2.undispatched_sweep import (+ UNDISPATCHED_ERROR_MESSAGE,+ sweep_undispatched_executions,+ )++ ex = _execution()+ assert sweep_undispatched_executions() == 1++ ex.refresh_from_db()+ assert ex.status == ExecutionStatus.ERROR.value+ assert ex.error_message == UNDISPATCHED_ERROR_MESSAGE+ assert (+ WorkflowExecution.objects.filter(status=ExecutionStatus.PENDING.value).count()+ == 0+ )++ def test_it_is_idempotent(self):+ """Runs on every tick — a second pass must find nothing, not re-write."""+ from workflow_manager.workflow_v2.undispatched_sweep import (+ sweep_undispatched_executions,+ )++ _execution()+ assert sweep_undispatched_executions() == 1+ assert sweep_undispatched_executions() == 0+++class TestNeverTouchesALiveExecution:+ """The dangerous direction. Each case is a way a live run could be killed."""++ def test_a_young_execution_is_left_alone(self):+ """Dispatch follows creation within seconds; age is the only separator."""+ from workflow_manager.workflow_v2.undispatched_sweep import (+ sweep_undispatched_executions,+ )++ ex = _execution(created_at=timezone.now() - timedelta(seconds=30))+ assert sweep_undispatched_executions() == 0+ ex.refresh_from_db()+ assert ex.status == ExecutionStatus.PENDING.value++ def test_a_pg_dispatched_execution_is_left_alone(self):+ """queue_message_id set => the PG queue holds it; the reaper owns it now."""+ from workflow_manager.workflow_v2.undispatched_sweep import (+ sweep_undispatched_executions,+ )++ ex = _execution(queue_message_id=4242)+ assert sweep_undispatched_executions() == 0+ ex.refresh_from_db()+ assert ex.status == ExecutionStatus.PENDING.value++ def test_a_celery_dispatched_execution_is_left_alone(self):+ """task_id set => Celery holds it. The window is upstream of+ resolve_transport, so the predicate must be transport-aware or a flag-off+ deployment would have its live executions terminalised.+ """+ from workflow_manager.workflow_v2.undispatched_sweep import (+ sweep_undispatched_executions,+ )++ ex = _execution(task_id=uuid.uuid4())+ assert sweep_undispatched_executions() == 0+ ex.refresh_from_db()+ assert ex.status == ExecutionStatus.PENDING.value++ @pytest.mark.parametrize(+ "status",+ [+ ExecutionStatus.EXECUTING.value,+ ExecutionStatus.COMPLETED.value,+ ExecutionStatus.ERROR.value,+ ExecutionStatus.STOPPED.value,+ ],+ )+ def test_only_pending_is_swept(self, status):+ """EXECUTING is running; the terminal three are already resolved. Rewriting+ a COMPLETED run's error_message would be corruption, not recovery.+ """+ from workflow_manager.workflow_v2.undispatched_sweep import (+ sweep_undispatched_executions,+ )++ ex = _execution(status=status)+ assert sweep_undispatched_executions() == 0+ ex.refresh_from_db()+ assert ex.status == status+ assert ex.error_message == ""++ def test_a_row_dispatched_mid_sweep_is_not_clobbered(self):+ """The read-then-write race, which is why the UPDATE re-carries the predicate.++ Simulates dispatch landing between candidate selection and the write: the+ row is passed in as a candidate id but no longer matches, so Postgres skips+ it. A plain `filter(id__in=ids).update(...)` would mark a live run ERROR.+ """+ from workflow_manager.workflow_v2.models import WorkflowExecution+ from workflow_manager.workflow_v2.undispatched_sweep import (+ sweep_undispatched_executions,+ )++ ex = _execution()+ WorkflowExecution.objects.filter(id=ex.id).update(queue_message_id=99)++ assert sweep_undispatched_executions() == 0+ ex.refresh_from_db()+ assert ex.status == ExecutionStatus.PENDING.value+++class TestDispatchIsAPositiveFactNotAnInference:+ """The fix for the defect the whole file's caution was working around.++ The sweep used to infer "never dispatched" from `task_id IS NULL AND+ queue_message_id IS NULL`. Three paths in workflow_helper reach that state on a+ RUNNING execution — a swallowed exception around _record_dispatch_handle, an empty+ handle, an unparseable PG handle — so the sweep could mark live work ERROR and, on+ the PG transport, get it silently DROPPED (both worker entry points stop on a+ terminal execution rather than superseding it).++ `dispatched_at` is stamped the instant dispatch succeeds, upstream of all three.+ """++ def test_a_dispatched_row_is_never_swept_even_with_no_handles(self):+ """THE regression. This is exactly the shape the three failure paths produce:+ dispatch succeeded, neither handle was recorded. Before dispatched_at this row+ was swept to ERROR while its worker was still running.+ """+ from workflow_manager.workflow_v2.undispatched_sweep import (+ sweep_undispatched_executions,+ )++ ex = _execution(dispatched_at=timezone.now() - timedelta(hours=2))+ assert sweep_undispatched_executions() == 0+ ex.refresh_from_db()+ assert ex.status == ExecutionStatus.PENDING.value++ def test_an_unstamped_row_with_a_handle_is_still_not_swept(self):+ """The single-phase rollout guarantee. During a rolling deploy an OLD backend+ pod dispatches without stamping, so the row has no dispatched_at — but it does+ have a handle. Keeping the handle checks alongside the new one is what lets this+ ship in one release instead of two.+ """+ from workflow_manager.workflow_v2.undispatched_sweep import (+ sweep_undispatched_executions,+ )++ ex = _execution(task_id=uuid.uuid4())+ assert sweep_undispatched_executions() == 0+ ex.refresh_from_db()+ assert ex.status == ExecutionStatus.PENDING.value++ def test_a_genuinely_undispatched_row_is_still_swept(self):+ """The fix must not disable the sweep. A row with no stamp and no handle is the+ 967-orphan case this exists for, and it must still terminalise.+ """+ from workflow_manager.workflow_v2.undispatched_sweep import (+ sweep_undispatched_executions,+ )++ ex = _execution()+ assert sweep_undispatched_executions() == 1+ ex.refresh_from_db()+ assert ex.status == ExecutionStatus.ERROR.value+++class TestGracePeriodIsSizedForTheQueue:+ """The grace period is the ONLY thing separating "abandoned" from "dispatched but+ not yet recorded", and on PG that distinction has teeth.++ A row this sweep terminalises while its message is still queued does not get run+ late — both PG worker entry points STOP on a terminal execution and ack the message+ (general/tasks.py returns skipped_terminal_execution; file_processing/tasks.py+ raises _TerminalExecutionSkip), so the work is dropped. On Celery the orchestrator+ ran regardless and its own terminal write superseded the ERROR, which is why this+ was survivable before the PG transport and is not after it.++ An hour is chosen to sit clear of any realistic dequeue latency. Lowering it back+ toward the queue's p99 wait re-opens that window, so this pins the value: change it+ deliberately, with a latency figure in hand, not as a tidy-up.+ """++ def test_the_grace_period_is_an_hour(self):+ from workflow_manager.workflow_v2.undispatched_sweep import (+ DEFAULT_MIN_AGE_SECONDS,+ )++ assert DEFAULT_MIN_AGE_SECONDS == 3600++ def test_it_stays_clear_of_the_barrier_stuck_timeout(self):+ """Must remain well under ~2.5h or this sweep and the barrier reaper start+ contending for the same rows — the disjointness the two sweeps depend on.+ """+ from workflow_manager.workflow_v2.undispatched_sweep import (+ DEFAULT_MIN_AGE_SECONDS,+ )++ assert DEFAULT_MIN_AGE_SECONDS < 9000+++class TestNeverDeletesStagedInput:+ """The sweep must not perform any IRREVERSIBLE cleanup, because its "never+ dispatched" test is an INFERENCE and the inference is unsound.++ Both handles being NULL is reached by three paths in ``workflow_helper`` AFTER the+ message is on its transport: ``_record_dispatch_handle`` raising and being swallowed+ by the caller ("continuing — the orchestrator is already running"), an empty handle+ returning early, and a PG handle that will not parse returning early. So a claimed+ row may be a LIVE execution.++ Marking it ERROR is survivable — the running worker's own terminal write supersedes+ it, and error→completed is explicitly permitted by the status guard. Deleting its+ staged input is not: the worker is still going to read it.++ This pins the ABSENCE of that call. Re-adding it is only safe once dispatch is a+ positive fact (a stamped ``dispatched_at``) rather than an inferred absence.+ """++ def test_the_sweep_module_does_not_delete_api_storage(self):+ import inspect++ from workflow_manager.workflow_v2 import undispatched_sweep++ source = inspect.getsource(undispatched_sweep)+ executable = "\n".join(+ line for line in source.splitlines() if not line.lstrip().startswith("#")+ )+ assert "delete_api_storage_dir" not in executable++ def test_releasing_resources_still_frees_the_rate_limit_slot(self):+ """The reversible half must survive the removal — a held slot consumes the+ org's API-deployment concurrency budget until the limiter TTL expires it.+ """+ import inspect++ from workflow_manager.workflow_v2 import undispatched_sweep++ source = inspect.getsource(undispatched_sweep._release_abandoned_resources)+ assert "release_slot" in source+++class TestUserFacingMessage:+ """`error_message` is rendered in the UI — ExecutionSerializer uses `exclude`,+ not `fields`, so every unlisted model field is serialized to customers.+ """++ def test_it_fits_the_column_with_the_ref_intact(self):+ """CharField(max_length=256) truncates silently, which would eat the trailing+ ref code and leave support without the greppable handle.+ """+ from workflow_manager.workflow_v2.models.execution import (+ EXECUTION_ERROR_LENGTH,+ )+ from workflow_manager.workflow_v2.undispatched_sweep import (+ UNDISPATCHED_ERROR_MESSAGE,+ )++ assert len(UNDISPATCHED_ERROR_MESSAGE) <= EXECUTION_ERROR_LENGTH+ assert UNDISPATCHED_ERROR_MESSAGE.rstrip().endswith("(ref: EXEC_NOT_STARTED)")++ def test_it_names_no_internals(self):+ """A customer reading this must not meet our vocabulary. The existing reaper+ strings ("the final aggregating callback never fired before the barrier+ expired") are the anti-pattern this guards against.+ """+ from workflow_manager.workflow_v2.undispatched_sweep import (+ UNDISPATCHED_ERROR_MESSAGE,+ )++ leaked = [+ w+ for w in (+ "queue",+ "barrier",+ "dispatch",+ "gateway",+ "celery",+ "worker",+ "reaper",+ "502",+ "null",+ )+ if w in UNDISPATCHED_ERROR_MESSAGE.lower()+ ]+ assert not leaked, f"internal term(s) in a user-facing message: {leaked}"++ def test_it_answers_what_the_user_needs(self):+ """Did anything run, is my data affected, what do I do."""+ from workflow_manager.workflow_v2.undispatched_sweep import (+ UNDISPATCHED_ERROR_MESSAGE,+ )++ low = UNDISPATCHED_ERROR_MESSAGE.lower()+ assert "did not start" in low+ assert "no files were processed" in low+ assert "run it again" in lowdiff --git a/backend/workflow_manager/workflow_v2/undispatched_sweep.py b/backend/workflow_manager/workflow_v2/undispatched_sweep.pynew file mode 100644index 000000000..cc3d77f35--- /dev/null+++ b/backend/workflow_manager/workflow_v2/undispatched_sweep.py@@ -0,0 +1,303 @@+"""Terminalise executions that were created but never dispatched to a queue.++``PENDING`` is not a terminal state, so every execution must eventually reach+``COMPLETED`` or ``ERROR``. One case had no owner:++ deployment_helper.py:236 create_workflow_execution(...) -> commits a PENDING row+ ... file handling, validation ...+ deployment_helper.py:310 execute_workflow_async(...) -> dispatches++An abort between those two — client disconnect, gateway timeout, pod eviction, OOM —+leaves a committed ``PENDING`` row that was never queued. Nothing recovers it:++* the **reaper** recovers stranded work by scanning ``pg_barrier_state``, and a barrier+ only exists once a batch has been dispatched. No dispatch, no barrier, invisible.+* ``execute_workflow_async`` marks *dispatch failures* ERROR, but never runs at all if+ the request dies before reaching it.++Observed on integration 2026-08-17: an 800-user load test saturated the API tier+(2 backend pods), 71% of requests got a 502 from the load balancer, and **967+executions were left PENDING** — 2914 execution rows from 2444 requests, because the+row outlived the request. ``pg_queue_message`` was completely empty, confirming the+work was never enqueued rather than enqueued-and-lost.++**Not PG-specific.** The create-then-dispatch ordering sits *upstream* of+``resolve_transport``, so the Celery path has the identical window. The predicate here+is transport-aware for exactly that reason.++**The predicate.** ``workflow_helper.py:566/570`` stamps ``queue_message_id`` (PG) or+``task_id`` (Celery) immediately after a successful dispatch, and the model documents+that the other stays NULL. So ``PENDING`` + both handles NULL + older than the grace+period means the dispatch USUALLY never happened — but see the note in+_release_abandoned_resources: three paths in workflow_helper reach that exact+state AFTER dispatch, which is why this sweep does no irreversible cleanup and+why the real fix is a positive dispatched_at rather than an inferred absence.+Do not shorten the grace period or reinstate cleanup on the strength of this+paragraph alone. Nominally it means the dispatch never happened — no JSONB probing, no cross-table joins, and+correct under either transport.+"""++from __future__ import annotations++import logging+import os++from django.utils import timezone++from unstract.core.data_models import ExecutionStatus++logger = logging.getLogger(__name__)++# Grace period before an undispatched execution is considered abandoned.+#+# Dispatch normally follows row creation within seconds, so this is generous by two+# orders of magnitude. It has to be: the ONLY thing separating "abandoned" from "about+# to be dispatched" is elapsed time, and terminalising a live execution is far worse+# than leaving a dead one a while longer. Well under the barrier stuck-timeout (~2.5h)+# so the two sweeps never contend for the same row.+# RAISED 900 -> 3600 deliberately, and the reason is PG-specific.+#+# The predicate cannot distinguish "never dispatched" from "dispatched, handle not+# recorded, message still sitting in the queue" — see the note in+# _release_abandoned_resources for the three paths that produce the second state. On+# CELERY that mismatch was survivable: the orchestrator runs on the row regardless and+# its terminal write supersedes the sweep's ERROR. On PG it is not. Both worker entry+# points STOP on a terminal execution (general/tasks.py returns+# skipped_terminal_execution; file_processing/tasks.py raises _TerminalExecutionSkip),+# so a row this sweep terminalises while its message is still queued gets acked and+# dropped rather than run.+#+# That failure needs a slow queue AND an unrecorded handle at once. An hour is chosen to+# sit well clear of any realistic dequeue latency, which is what shrinks the overlap to+# near-nothing without new machinery. It is a mitigation, not the fix: the fix is to make+# dispatch a POSITIVE fact (a stamped dispatched_at) so the predicate stops guessing.+#+# Still well under the barrier stuck-timeout (~2.5h), so the two sweeps never contend for+# the same row. Cost of the raise: a genuinely undispatched execution shows PENDING for+# up to an hour before it errors. Override per-environment with the env var if a+# deployment's queue latency justifies something tighter.+_MIN_AGE_ENV = "UNDISPATCHED_EXECUTION_GRACE_SECONDS"+DEFAULT_MIN_AGE_SECONDS = 3600 # 1 hour++# Bounds one sweep so a large backlog (a 502 storm leaves hundreds) can't hold a long+# transaction open. Whatever is left is picked up by the next tick.+_BATCH_LIMIT_ENV = "UNDISPATCHED_EXECUTION_SWEEP_LIMIT"+DEFAULT_BATCH_LIMIT = 500++# USER-FACING. `error_message` is rendered in the UI: ExecutionSerializer uses+# `exclude`, not `fields`, so every unlisted model field is serialized — this string+# reaches customers. It answers their three questions (did anything run? is my data+# affected? what do I do?) and deliberately names no internals: no queues, barriers,+# dispatch or gateways. The machine-readable ref is for support/greppability, and the+# precise technical cause stays in the log line below.+#+# Must fit EXECUTION_ERROR_LENGTH (256); the model truncates silently, which would+# otherwise eat the ref code at the end.+UNDISPATCHED_ERROR_MESSAGE = (+ "This execution did not start. The request was interrupted before any processing "+ "began, so no files were processed. You can safely run it again. "+ "(ref: EXEC_NOT_STARTED)"+)+++def _positive_int_from_env(name: str, default: int) -> int:+ """Env override, falling back loudly rather than silently on a bad value.++ A shortened grace period is the dangerous direction (it terminalises live+ executions), so a typo must not quietly take effect.+ """+ raw = os.environ.get(name)+ if raw is None:+ return default+ try:+ value = int(raw)+ except (TypeError, ValueError):+ logger.warning("%s=%r is not an integer; using default %s", name, raw, default)+ return default+ if value <= 0:+ logger.warning("%s=%r must be > 0; using default %s", name, raw, default)+ return default+ return value+++# Claim the batch in ONE statement that also reports exactly which rows it won.+#+# `UPDATE ... RETURNING` is the whole design in a single round trip:+# * the inner SELECT bounds the batch and takes row locks (SKIP LOCKED, so an+# overlapping sweeper can never block or double-claim);+# * the OUTER WHERE re-carries the full predicate, so Postgres re-evaluates it at+# write time under those locks — a row dispatched between selection and write is+# silently skipped rather than marked ERROR *while it runs*, which is the one+# failure this sweep must never cause;+# * RETURNING yields precisely the claimed rows, which is what lets the irreversible+# cleanup run for those and only those. (An earlier draft looped one UPDATE per+# row to get that guarantee; this keeps it and drops 500 round trips.)+#+# Django's .update() cannot RETURN rows, hence raw SQL — the same approach the PG-queue+# reaper's own sweeps use. The table name is interpolated from _meta (never user input).+_CLAIM_SQL = """+UPDATE {table}+ SET status = %s, error_message = %s, modified_at = %s+ WHERE id IN (+ SELECT id+ FROM {table}+ WHERE status = %s+ AND created_at < %s+ AND dispatched_at IS NULL+ AND task_id IS NULL+ AND queue_message_id IS NULL+ ORDER BY created_at+ LIMIT %s+ FOR UPDATE SKIP LOCKED+ )+ AND status = %s+ AND dispatched_at IS NULL+ AND task_id IS NULL+ AND queue_message_id IS NULL+RETURNING id, workflow_id+"""+++def sweep_undispatched_executions(+ min_age_seconds: int | None = None, limit: int | None = None+) -> int:+ """Mark aged, never-dispatched PENDING executions ERROR. Returns the count.++ Idempotent and safe to run on every sweep: a row that has since been dispatched or+ terminalised no longer matches the predicate, so a second pass finds nothing.++ Race-free by construction — see :data:`_CLAIM_SQL`.+ """+ from django.db import connection++ from workflow_manager.workflow_v2.models import WorkflowExecution++ min_age = min_age_seconds or _positive_int_from_env(+ _MIN_AGE_ENV, DEFAULT_MIN_AGE_SECONDS+ )+ batch_limit = limit or _positive_int_from_env(_BATCH_LIMIT_ENV, DEFAULT_BATCH_LIMIT)+ now = timezone.now()+ cutoff = now - timezone.timedelta(seconds=min_age)++ sql = _CLAIM_SQL.format(table=WorkflowExecution._meta.db_table)+ with connection.cursor() as cursor:+ cursor.execute(+ sql,+ [+ ExecutionStatus.ERROR.value,+ UNDISPATCHED_ERROR_MESSAGE,+ now,+ ExecutionStatus.PENDING.value,+ cutoff,+ batch_limit,+ ExecutionStatus.PENDING.value,+ ],+ )+ claimed = cursor.fetchall()++ if not claimed:+ return 0++ for execution_id, workflow_id in claimed:+ _release_abandoned_resources(str(execution_id), str(workflow_id))++ # The operator-facing half of the story. The user-facing column says only what a+ # customer needs; the cause belongs here, where it can name internals freely.+ logger.error(+ "Undispatched-execution sweep: marked %s execution(s) ERROR. They were created "+ "but never queued (task_id and queue_message_id both NULL) and exceeded the %ss "+ "grace period — the request died between create_workflow_execution and "+ "execute_workflow_async. Ids: %s",+ len(claimed),+ min_age,+ ", ".join(str(row[0]) for row in claimed[:20]),+ )+ return len(claimed)+++def _release_abandoned_resources(execution_id: str, workflow_id: str) -> None:+ """Do what the abort prevented the request's own error path from doing.++ ``deployment_helper`` releases the rate-limit slot and deletes the API storage dir+ when staging *raises*. An abort raises nothing — the thread is simply gone — so+ neither runs and the execution leaks both.++ **Best-effort, and deliberately so.** The status write already succeeded and is the+ part that matters; a failure to tidy up must never propagate and stall the rest of+ the batch.++ Only the slot is released here. The staged input is deliberately retained — see the+ note in the body for why, and why re-adding that delete needs the predicate fixed+ first.+ """+ # Slot first: it is the one with a live cost. Held slots consume the org's API+ # deployment concurrency budget until the Redis ZSET TTL (6h) expires them, so a+ # 502 storm can throttle a tenant for hours. Self-healing, but slowly.+ try:+ from api_v2.rate_limiter import APIDeploymentRateLimiter++ from workflow_manager.workflow_v2.models import WorkflowExecution++ organization = (+ WorkflowExecution.objects.select_related("workflow__organization")+ .get(id=execution_id)+ .workflow.organization+ )+ # str(...organization_id), NOT the model instance. release_slot builds its Redis+ # key by formatting the argument into "api_deployment:rate_limit:org:{org_id}",+ # and acquire_slot built it from str(organization.organization_id). Passing the+ # instance yields "...:org:Organization object (12)", which matches nothing — the+ # ZREM removes a non-member, returns 0, raises nothing, and the slot stays held+ # for the full 6h TTL. Silent, and the exact harm this call exists to prevent.+ APIDeploymentRateLimiter.release_slot(+ str(organization.organization_id), execution_id+ )+ except Exception:+ logger.warning(+ "Undispatched sweep: could not release the rate-limit slot for %s "+ "(it expires with the limiter TTL regardless)",+ execution_id,+ exc_info=True,+ )++ # The staged input is deliberately NOT deleted here.+ #+ # This sweep infers "never dispatched" from the ABSENCE of both handles+ # (task_id IS NULL AND queue_message_id IS NULL). That inference is not sound:+ # three paths in workflow_helper leave a RUNNING execution in exactly that state,+ # all of them after the message is already on its transport —+ #+ # * _record_dispatch_handle raises and the caller swallows it, explicitly+ # "continuing — the orchestrator is already running" (workflow_helper.py:686)+ # * the handle comes back empty and it returns without writing (:544)+ # * a PG handle will not parse as a bigint and it returns without writing (:562)+ #+ # so after the 15-minute grace this sweep can claim a live execution.+ #+ # Do NOT read the surviving ERROR write as harmless. An earlier version of this note+ # claimed it "self-corrects — the running worker's own terminal write supersedes it".+ # That is false on the PG transport: both worker entry points treat a terminal+ # execution as a reason to STOP, not to overwrite. general/tasks.py returns+ # skipped_terminal_execution before it would write EXECUTING, and+ # file_processing/tasks.py raises _TerminalExecutionSkip. So a wrongly-claimed row is+ # silently DROPPED — the message is acked, no retry is scheduled, and the only trace+ # is a worker WARNING. The realistic trigger is not the three no-handle paths above+ # but a saturated queue: a message enqueued successfully and still unconsumed at 15+ # minutes gets marked ERROR and then discarded, under exactly the backlog conditions+ # that produced the orphans this sweep was built for.+ #+ # Even where a terminal write does land, error_message is not cleared by it, so a+ # wrongly-claimed run can complete while still showing the customer "This execution+ # did not start... You can safely run it again."+ #+ # Deleting the input is worse again and is what this removal addresses: the worker is+ # still going to read it, and the message invites a re-run against data that is gone.+ #+ # Dropping the delete keeps the reversible half of the sweep and removes the+ # irreversible one, at the cost of leaking an input directory for executions that+ # genuinely never started. That is the right trade while the predicate is unsound;+ # the real fix is to make dispatch a POSITIVE fact (stamp dispatched_at in the same+ # call that records the handle, including on the three branches above, and key both+ # this predicate and the 0026 partial index on it) — tracked separately, since it+ # needs a migration and an index rebuild.+ _ = workflow_id # retained: the signature is restored with the positive-fact fixdiff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex d22dcc85b..971fb7a8c 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -666,6 +666,28 @@ class WorkflowHelper:# the bookkeeping below must NOT flip the (now-running) row to ERROR.dispatched = True+ # Record dispatch as a POSITIVE fact, before any bookkeeping that can+ # fail. The undispatched sweep used to infer "never dispatched" from+ # `task_id IS NULL AND queue_message_id IS NULL`, which the three paths+ # below all reach on a RUNNING execution — the swallowed exception around+ # _record_dispatch_handle, and its two early returns for an empty or+ # unparseable handle. This single write sits upstream of all three, so the+ # sweep can ask a question with a true answer instead of guessing.+ #+ # Its own try/except for the same reason the handle write has one: we are+ # past the point of no return and nothing here may abort the caller. If it+ # fails we are back to the old ambiguity for this one execution, which is+ # strictly no worse than before.+ try:+ WorkflowExecutionServiceHelper.mark_dispatched(execution_id)+ except Exception:+ logger.exception(+ f"[{org_schema}] Failed to stamp dispatched_at for execution "+ f"'{execution_id}'; continuing — the orchestrator is already "+ "running. This row is now indistinguishable from an undispatched "+ "one and may be terminalised by the undispatched sweep."+ )+workflow_execution: WorkflowExecution = WorkflowExecution.objects.get(id=execution_id)@@ -1033,6 +1055,37 @@ class WorkflowHelper:)try:workflow_execution = WorkflowExecution.objects.get(pk=execution_id)+ # Single-step is the one entry path that never moved onto PG. Its+ # only fan-out is the Celery chord in `process_input_files` — the+ # normal path's PG fan-out lives in the general worker instead, so+ # nothing here is transport-gated. With the flag on and the Celery+ # file_processing workers scaled to zero (the epic's acceptance+ # gate), those batches would sit unconsumed and the execution would+ # hang in EXECUTING, invisible to the PG reaper. Fail fast instead:+ # a 500 with a stated cause beats a silent forever-EXECUTING row.+ #+ # Unreachable from the UI — the step buttons are gone and both live+ # call sites pass isStepExecution=false, so `execution_action` is+ # never sent (frontend Agency.jsx). This guards a direct API call.+ #+ # Resolved here rather than in `process_input_files` so the Celery+ # path stays byte-identical and an already-resolved execution is+ # never re-resolved. Step executions are created by+ # `create_and_make_execution_response`, which resolves no transport,+ # so this is their first and only resolution.+ transport = resolve_transport(+ execution_id=execution_id,+ organization_id=workflow.organization.organization_id,+ workflow_id=workflow.id,+ )+ if is_pg_transport(transport):+ raise WorkflowExecutionError(+ "Single-step execution is not supported on the Postgres "+ "queue transport: it has no PG fan-out, and its Celery "+ "batches would never be consumed. Run the full workflow "+ "instead, or disable pg_queue_enabled for this "+ "organization."+ )return WorkflowHelper.run_workflow(workflow=workflow,single_step=True,diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yamlindex 10a7b4e5d..dda8f27a2 100644--- a/docker/docker-compose.yaml+++ b/docker/docker-compose.yaml@@ -742,6 +742,93 @@ services:profiles:- pg-queue+ # Dashboard-metrics periodics over PG (UN-3796) — the execution half of the Beat+ # replacement. The PG scheduler fires dashboard_metrics.* onto dashboard_metric_events+ # and this drains them, replacing the Celery worker-metrics+ # (-A backend worker -Q dashboard_metric_events).+ #+ # A dedicated service rather than another queue on worker-pg-scheduler: the+ # consumer's health heartbeat is stamped per poll and FROZEN while a task runs, so+ # HEALTH_STALE doubles as an upper bound on one task's wall clock. The aggregation+ # runs for minutes; under the scheduler's 240s bound it would trip the liveness+ # probe, restart the pod, and take in-flight pipeline triggers down with it.+ #+ # No rabbitmq dependency — the tasks are thin HTTP proxies to the backend internal+ # API and dispatch nothing onward. MAX_ATTEMPTS=1 because for a periodic the next+ # cron tick supersedes a failed run; retrying would let a persistently failing+ # aggregation stack up in-flight messages.+ worker-pg-metrics:+ image: unstract/worker-unified:${VERSION}+ container_name: unstract-worker-pg-metrics+ restart: unless-stopped+ # The GENERIC consumer command, with identity carried by the env below — the+ # container entrypoint (run-worker-docker.sh) is env-driven and its `pg-*`+ # dispatch accepts only pg-queue-consumer / pg-consumer / pg-queue-reaper /+ # pg-reaper / reaper, rejecting anything else with exit 1. The named+ # `pg-metrics` role is for the HOST runner (`./run-worker.sh pg-metrics`),+ # which is a different script and does understand roles.+ command: ["pg-queue-consumer"]+ ports:+ - "8101:8090"+ env_file:+ - ../workers/.env+ - ./essentials.env+ depends_on:+ - db+ - redis+ environment:+ - ENVIRONMENT=development+ - APPLICATION_NAME=unstract-worker-pg-metrics+ - WORKER_BARRIER_BACKEND=pg+ - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=scheduler+ - WORKER_PG_QUEUE_CONSUMER_QUEUE=dashboard_metric_events+ - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090+ # Sized for a minutes-long aggregation: stale > vt > the task's own ceiling.+ - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_METRICS_VT_SECONDS:-900}+ - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_METRICS_HEALTH_STALE_SECONDS:-960}+ # One at a time — these are global singletons, not parallel work.+ - WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=1+ - WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS=1+ labels:+ - traefik.enable=false+ profiles:+ - pg-queue++ # Log streaming without Celery (UN-3755) — the flag-on replacement for+ # worker-log-consumer-v2. Note the transport here is a REDIS LIST, not+ # pg_queue_message: the log consumer's own sinks are already Redis (log_history_queue+ # + the Socket.IO emit), so routing through Postgres would detour a lossy, high-rate+ # stream through the platform's most contended resource to buy durability that logs+ # explicitly discard at their 10k cap. It sits in the pg-queue profile because it is+ # part of the same flag-on fleet, not because it speaks to Postgres.+ #+ # LOG_TRANSPORT must be set to "redis" on every PUBLISHER too (backend, workers,+ # runner) or this drains an empty list while logs keep going to RabbitMQ.+ worker-log-stream-consumer:+ image: unstract/worker-unified:${VERSION}+ container_name: unstract-worker-log-stream-consumer+ restart: unless-stopped+ command: ["log-stream-consumer"]+ env_file:+ - ../workers/.env+ - ./essentials.env+ depends_on:+ - redis+ environment:+ - ENVIRONMENT=development+ - APPLICATION_NAME=unstract-worker-log-stream-consumer+ - WORKER_TYPE=log_consumer+ - WORKER_NAME=log-stream-consumer+ - LOG_TRANSPORT=redis+ - LOG_STREAM_QUEUE_NAME=${LOG_STREAM_QUEUE_NAME:-log_stream_queue}+ # Downstream sinks are unchanged — same durable buffer the Celery worker wrote to.+ - LOG_HISTORY_QUEUE_NAME=${LOG_HISTORY_QUEUE_NAME:-log_history_queue}+ - ENABLE_LOG_HISTORY=${ENABLE_LOG_HISTORY:-true}+ labels:+ - traefik.enable=false+ profiles:+ - pg-queue+# Executor RPC over PG — runs execute_extraction as a request-reply: claims# work from Postgres and writes the ExecutionResult to pg_task_result for the# blocking caller. Same heavy executor runtime as worker-executor-v2 (tooldiff --git a/docker/sample.env b/docker/sample.envindex e5e1ca7c6..6c96821a4 100644--- a/docker/sample.env+++ b/docker/sample.env@@ -133,6 +133,17 @@ ENABLE_METRICS=true# WORKER_PG_REAPER_HEALTH_PORT - opt-in reaper liveness port (unset = off).WORKER_PG_REAPER_INTERVAL_SECONDS=5 # Reaper sweep interval (seconds)#+# worker-pg-metrics (UN-3796) — drains the dashboard_metrics.* periodics the PG+# scheduler fires, replacing Celery Beat + worker-metrics. Its knobs are sized for a+# task that runs for MINUTES, unlike every other consumer here:+#+# WORKER_PG_METRICS_VT_SECONDS - claim/drain bound (default 900).+# WORKER_PG_METRICS_HEALTH_STALE_SECONDS- liveness bound (default 960). The heartbeat+# freezes while a task runs, so this is effectively an+# upper bound on ONE task's wall clock — it must stay+# above the vt, which must stay above the task itself.+# above the vt, which must stay above the task itself.+## NOTE: routing executions to PG is a SEPARATE, later step. Running these# services does NOT move any traffic — the single `pg_queue_enabled` Flipt flag# (default off, fail-closed) decides per-execution transport, and stays off untildiff --git a/runner/src/unstract/runner/constants.py b/runner/src/unstract/runner/constants.pyindex b46971d61..e4f7449fb 100644--- a/runner/src/unstract/runner/constants.py+++ b/runner/src/unstract/runner/constants.py@@ -44,3 +44,10 @@ class Env:CELERY_BROKER_BASE_URL = "CELERY_BROKER_BASE_URL"CELERY_BROKER_USER = "CELERY_BROKER_USER"CELERY_BROKER_PASS = "CELERY_BROKER_PASS"+ # Log-streaming transport (UN-3755). Must reach the SIDECAR: it is the process+ # that calls LogPublisher.publish for tool logs+ # (tool_sidecar/log_processor.py:165), and the sidecar's environment is a+ # hand-picked allowlist rather than an inherited one — so anything absent here is+ # silently absent there, and it falls back to publishing on Celery/RabbitMQ.+ LOG_TRANSPORT = "LOG_TRANSPORT"+ LOG_STREAM_QUEUE_NAME = "LOG_STREAM_QUEUE_NAME"diff --git a/runner/src/unstract/runner/runner.py b/runner/src/unstract/runner/runner.pyindex 1125bf69e..2414ac754 100644--- a/runner/src/unstract/runner/runner.py+++ b/runner/src/unstract/runner/runner.py@@ -245,6 +245,16 @@ class UnstractRunner:"CELERY_BROKER_BASE_URL": os.getenv(Env.CELERY_BROKER_BASE_URL),"CELERY_BROKER_USER": os.getenv(Env.CELERY_BROKER_USER),"CELERY_BROKER_PASS": os.getenv(Env.CELERY_BROKER_PASS),+ # Log transport (UN-3755). The sidecar is a LogPublisher producer+ # (log_processor.py:165), so it must agree with every other publisher or+ # its tool logs go to RabbitMQ while the rest go to Redis — and with the+ # Celery log consumer scaled to zero, those logs are simply lost. This+ # dict is an allowlist, not inherited env, so omitting it fails silently.+ # Needs no new credentials: REDIS_* is already passed above.+ Env.LOG_TRANSPORT: os.getenv(Env.LOG_TRANSPORT, "celery"),+ Env.LOG_STREAM_QUEUE_NAME: os.getenv(+ Env.LOG_STREAM_QUEUE_NAME, "log_stream_queue"+ ),"CONTAINER_NAME": container_name,}sidecar_config = self.client.get_container_run_config(diff --git a/runner/tests/test_sidecar_log_transport.py b/runner/tests/test_sidecar_log_transport.pynew file mode 100644index 000000000..01c46280b--- /dev/null+++ b/runner/tests/test_sidecar_log_transport.py@@ -0,0 +1,84 @@+"""The sidecar must inherit the log transport (UN-3755).++``_get_sidecar_container_config`` builds the sidecar's environment as a hand-picked+**allowlist**, not inherited env. So a variable the sidecar needs but that nobody added+here is silently absent at runtime — no error, no warning.++That matters for ``LOG_TRANSPORT`` specifically: the sidecar is a ``LogPublisher``+producer (``tool_sidecar/log_processor.py:165``), and ``LogPublisher`` defaults to the+Celery/RabbitMQ transport when the variable is unset. Miss it and tool logs keep going+to ``celery_log_task_queue`` while every other publisher moves to Redis — and once the+Celery log consumer is scaled to zero those logs are simply dropped, with live+streaming and ``execution_log`` rows both silently missing for container-based tools.+"""++from __future__ import annotations++from unittest.mock import MagicMock++import pytest++from unstract.runner.constants import Env+from unstract.runner.runner import UnstractRunner+++def sidecar_env(monkeypatch):+ """Build a sidecar env dict with the client mocked out."""++ def _build(**env):+ for key, value in env.items():+ monkeypatch.setenv(key, value)+ runner = UnstractRunner.__new__(UnstractRunner)+ runner.client = MagicMock()+ runner.client.get_container_run_config.side_effect = (+ lambda **kwargs: kwargs # return the call kwargs so we can read `envs`+ )+ config = runner._get_sidecar_container_config(+ container_name="c",+ shared_log_dir="/d",+ shared_log_file="/d/log",+ organization_id="org",+ execution_id="exec",+ file_execution_id="fe",+ messaging_channel="chan",+ tool_instance_id="ti",+ )+ return config["envs"]++ return _build+++class TestSidecarLogTransport:+ def test_defaults_to_celery_when_unset(self, sidecar_env, monkeypatch):+ monkeypatch.delenv(Env.LOG_TRANSPORT, raising=False)+ envs = sidecar_env()+ # Present and explicit, not merely absent — the sidecar's LogPublisher reads+ # this key, and an explicit default documents the flag-off state.+ assert envs[Env.LOG_TRANSPORT] == "celery"++ def test_forwards_redis_transport_to_the_sidecar(self, sidecar_env):+ envs = sidecar_env(LOG_TRANSPORT="redis")+ assert envs[Env.LOG_TRANSPORT] == "redis"++ def test_forwards_the_stream_queue_name(self, sidecar_env):+ # If a deployment renames the queue, the sidecar must push to the same list+ # the consumer drains, or its logs land somewhere nobody reads.+ envs = sidecar_env(LOG_TRANSPORT="redis", LOG_STREAM_QUEUE_NAME="custom_stream")+ assert envs[Env.LOG_STREAM_QUEUE_NAME] == "custom_stream"++ def test_redis_credentials_are_already_present(self, sidecar_env):+ """The Redis transport needs no NEW credential in this allowlist.++ This is the concrete payoff of choosing a Redis list over the PG queue for the+ log hop: REDIS_* is already forwarded, whereas PG would have required adding+ database credentials to every spawned sidecar.+ """+ envs = sidecar_env(LOG_TRANSPORT="redis", REDIS_HOST="r", REDIS_PORT="6379")+ assert envs["REDIS_HOST"] == "r"+ assert envs["REDIS_PORT"] == "6379"++ def test_celery_broker_still_forwarded_for_the_flag_off_path(self, sidecar_env):+ # Flag-off must stay intact: the sidecar still publishes over AMQP.+ envs = sidecar_env(CELERY_BROKER_BASE_URL="amqp://x")+ assert envs["CELERY_BROKER_BASE_URL"] == "amqp://x"diff --git a/tests/groups.yaml b/tests/groups.yamlindex 0fef1e82a..d2c35e34d 100644--- a/tests/groups.yaml+++ b/tests/groups.yaml@@ -111,6 +111,21 @@ groups:install_editable: truecoverage_source: src+ # runner/tests was registered in NO group, so it was collected by nothing. Its+ # only suite guards that LOG_TRANSPORT and LOG_STREAM_QUEUE_NAME reach the tool+ # sidecar's hand-picked env allowlist — a list that drops anything not named in+ # it, so a missing entry means the sidecar silently logs to the wrong transport.+ # Nothing else covers that, and it went in alongside the Redis log transport.+ # install_editable because the suite imports `unstract.runner`, which resolves+ # only from this package's own src layout (packages = ["src/unstract"]).+ unit-runner:+ tier: unit+ workdir: runner+ paths: [tests]+ uv_sync_group: test+ install_editable: true+ coverage_source: src+# ── Integration tier: needs infra but not full platform ────────────────────integration-backend:tier: integrationdiff --git a/unstract/core/src/unstract/core/data_models.py b/unstract/core/src/unstract/core/data_models.pyindex eba226743..bc2fd6dbc 100644--- a/unstract/core/src/unstract/core/data_models.py+++ b/unstract/core/src/unstract/core/data_models.py@@ -286,13 +286,23 @@ class QueueMessageState(StrEnum):set, mirroring the ``priority`` (fairness) precedent. ``str`` Enum → serialisesto its value and compares equal to the bare string.- - ``READY`` — claimable: the dequeue's partial claim index holds only these.- - ``CLAIMED`` — in-flight: a consumer holds it, ``vt`` is its renewable lease;+ - ``READY`` — claimable: the dequeue's partial claim index holds only these.+ - ``CLAIMED`` — in-flight: a consumer holds it, ``vt`` is its renewable lease;re-armed back to ``READY`` by the reaper when the lease expires (crash).+ - ``SCHEDULED`` — deferred (UN-3843): enqueued with a future ``available_at``+ (Celery ``countdown``/``eta`` parity) and **deliberately absent from the claim+ index**, so a not-yet-due row costs the hot claim path nothing. The reaper+ promotes it to ``READY`` once ``available_at <= now()``+ (``reaper.promote_due_scheduled``), which is why delivery is "not before+ ``available_at``" rather than exactly at it — granularity is the reaper tick.+ Filtering ``available_at`` in the claim instead would put every not-yet-due row+ back inside ``pg_queue_message_claim_idx`` to be walked and discarded on every+ claim — precisely the scan-past cost this enum exists to remove."""READY = "ready"CLAIMED = "claimed"+ SCHEDULED = "scheduled"# Fairness L3 priority bounds (1..10, higher = claimed sooner). Single source ofdiff --git a/unstract/core/src/unstract/core/pubsub_helper.py b/unstract/core/src/unstract/core/pubsub_helper.pyindex 3bec8475e..d42c48718 100644--- a/unstract/core/src/unstract/core/pubsub_helper.py+++ b/unstract/core/src/unstract/core/pubsub_helper.py@@ -12,6 +12,35 @@ from kombu import Connectionfrom unstract.core.cache.redis_client import create_redis_clientfrom unstract.core.constants import LogEventArgument, LogProcessingTask+#: Transport for the log-streaming hop between this publisher and the log consumer.+#: ``celery`` (default) publishes a Celery-protocol message to RabbitMQ; ``redis``+#: RPUSHes onto a Redis list drained by the PG-era log consumer.+#:+#: Deliberately an env var rather than the ``pg_queue_enabled`` Flipt flag. The flag+#: routes *per execution*, which is meaningless here: the consumer is a single+#: deployment reading from exactly one place, so a per-org split would strand half the+#: logs in a queue nobody drains. Producer and consumer must agree cluster-wide, which+#: makes this deployment config. It must be flipped in step with the flag rollout.+_LOG_TRANSPORT_ENV = "LOG_TRANSPORT"+_LOG_TRANSPORT_REDIS = "redis"++#: Redis list used when ``LOG_TRANSPORT=redis``. Distinct from ``log_history_queue``,+#: which is the *downstream* durable buffer the consumer writes to — this one is the+#: transport itself, and is normally near-empty.+_LOG_STREAM_QUEUE_ENV = "LOG_STREAM_QUEUE_NAME"+_LOG_STREAM_QUEUE_DEFAULT = "log_stream_queue"++#: Cap mirroring ``store_execution_log``'s (unstract/core/log_utils.py): without it a+#: stopped consumer grows this list until Redis OOMs, which would take down far more+#: than logging. Dropping is the correct behaviour for this payload — logs are already+#: best-effort and are discarded at the same cap one hop downstream.+_LOG_STREAM_MAX_SIZE_ENV = "LOG_STREAM_QUEUE_MAX_SIZE"+_LOG_STREAM_MAX_SIZE_DEFAULT = 10000+++def _use_redis_log_transport() -> bool:+ return os.getenv(_LOG_TRANSPORT_ENV, "celery").strip().lower() == _LOG_TRANSPORT_REDIS+class LogPublisher:broker_url = str(@@ -160,27 +189,30 @@ class LogPublisher:"""Publish a message to the queue."""try:event = f"logs:{channel_id}"- with cls.kombu_conn.Producer(serializer="json") as producer:- task_message = cls._get_task_message(- user_session_id=channel_id,- event=event,- message=payload,- )- headers = cls._get_task_header(LogProcessingTask.TASK_NAME)- # Publish the message to the queue- producer.publish(- body=task_message,- exchange="",- headers=headers,- routing_key=LogProcessingTask.QUEUE_NAME,- compression=None,- retry=True,- )- logging.debug(f"Published '{channel_id}' <= {payload}")-- # Persisting messages for unified notification- if payload.get("type") == "LOG":- cls.store_for_unified_notification(event, payload)+ task_message = cls._get_task_message(+ user_session_id=channel_id,+ event=event,+ message=payload,+ )+ if _use_redis_log_transport():+ cls._publish_via_redis(task_message)+ else:+ with cls.kombu_conn.Producer(serializer="json") as producer:+ headers = cls._get_task_header(LogProcessingTask.TASK_NAME)+ # Publish the message to the queue+ producer.publish(+ body=task_message,+ exchange="",+ headers=headers,+ routing_key=LogProcessingTask.QUEUE_NAME,+ compression=None,+ retry=True,+ )+ logging.debug(f"Published '{channel_id}' <= {payload}")++ # Persisting messages for unified notification+ if payload.get("type") == "LOG":+ cls.store_for_unified_notification(event, payload)except Exception as e:logging.error(f"Failed to publish '{channel_id}' <= {payload}"@@ -189,6 +221,42 @@ class LogPublisher:return Falsereturn True+ @classmethod+ def _publish_via_redis(cls, task_message: dict[str, Any]) -> None:+ """RPUSH the log envelope onto the Redis transport list.++ The envelope carries the task name alongside the kwargs so the consumer+ dispatches by name exactly as the Celery header did — a bare kwargs blob would+ leave the consumer guessing, and a name mismatch is the silent failure mode+ (message read, no handler, dropped with nothing at the publish site to trace).++ Raises on Redis failure; ``publish()`` owns the swallow, so a logging fault can+ never break an execution.+ """+ queue_name = os.getenv(_LOG_STREAM_QUEUE_ENV, _LOG_STREAM_QUEUE_DEFAULT)+ max_size = int(+ os.getenv(_LOG_STREAM_MAX_SIZE_ENV, str(_LOG_STREAM_MAX_SIZE_DEFAULT))+ )+ redis_client = cls._get_redis_client()++ # O(1), and the same llen-then-push shape store_execution_log already uses one+ # hop downstream. Two Redis round trips per log line is the price of not letting+ # a stopped consumer OOM Redis.+ if redis_client.llen(queue_name) >= max_size:+ logging.warning(+ f"Log stream queue '{queue_name}' at capacity ({max_size}), "+ "dropping current log - log consumer may be down or falling behind"+ )+ return++ envelope = json.dumps(+ {+ "task": LogProcessingTask.TASK_NAME,+ "kwargs": task_message["kwargs"],+ }+ )+ redis_client.rpush(queue_name, envelope)+@classmethoddef store_for_unified_notification(cls, event: str, payload: dict[str, Any]) -> None:"""Helps persist messages for unified notification.diff --git a/workers/log_consumer/redis_stream_consumer.py b/workers/log_consumer/redis_stream_consumer.pynew file mode 100644index 000000000..a2242f11f--- /dev/null+++ b/workers/log_consumer/redis_stream_consumer.py@@ -0,0 +1,163 @@+"""Redis-list consumer for the log stream (UN-3755).++Drains the list ``LogPublisher.publish()`` writes to when ``LOG_TRANSPORT=redis`` and+runs the **existing** ``logs_consumer`` task body against each envelope. Nothing about+what a log *does* changes here: the task still RPUSHes to ``log_history_queue`` and+emits over Socket.IO, and the log-history scheduler still drains that to+``execution_log``. This module replaces only the transport hop that RabbitMQ used to+provide, so the log consumer no longer needs a Celery deployment.++**Why a reliable-queue pattern rather than a bare BLPOP.** The Celery consumer runs with+``task_acks_late=True`` (``shared/infrastructure/config/worker_config.py:545``, and+``backend/celery_config.py:63``), so a worker that dies mid-task gets the message+redelivered. ``BLPOP`` deletes on read, which would silently make crashes lossy — a real+regression, not a no-op. ``BLMOVE`` instead parks the envelope on a per-pod *processing*+list and removes it only after the handler returns, and startup re-queues whatever the+previous incarnation left behind.++Bound worth knowing: recovery is keyed on ``HOSTNAME``, so a **container restart** (crash+loop, OOM kill — the dominant failure mode) recovers fully, while a **pod replacement**+strands that pod's in-flight envelopes, at most one per concurrent loop. That is strictly+better than BLPOP and, for a stream already discarded wholesale at its 10k cap, close+enough to the Celery behaviour it replaces. Sweeping other pods' lists is deliberately not+attempted: with multiple replicas it cannot distinguish a dead owner from a live one, and+would duplicate log lines on every start.+"""++from __future__ import annotations++import json+import os+import signal+import socket+import sys+from types import FrameType+from typing import Any++from shared.enums.worker_enums import WorkerType+from shared.infrastructure.config.builder import WorkerBuilder+from shared.infrastructure.logging import WorkerLogger++from unstract.core.cache.redis_client import create_redis_client+from unstract.core.constants import LogProcessingTask++logger = WorkerLogger.setup(WorkerType.LOG_CONSUMER)++# Build the Celery app before importing tasks: ``@worker_task`` is ``shared_task``, which+# binds to the current app at call time. Same ordering as ``log_consumer/worker.py``.+app, config = WorkerBuilder.build_celery_app(WorkerType.LOG_CONSUMER)++from log_consumer.tasks import logs_consumer # noqa: E402++_QUEUE_NAME = os.getenv("LOG_STREAM_QUEUE_NAME", "log_stream_queue")+# BLMOVE blocks up to this long before returning None, which is the loop's only chance to+# notice a shutdown signal. Keep it well under the pod's terminationGracePeriodSeconds.+_BLOCK_TIMEOUT_SECONDS = int(os.getenv("LOG_STREAM_BLOCK_TIMEOUT", "5"))+# redis-py enforces ``socket_timeout`` on the BLMOVE read itself, so it MUST exceed the+# server-side block or every call aborts mid-block with ``redis.TimeoutError``.+# ``create_redis_client`` defaults it to 5s — exactly ``_BLOCK_TIMEOUT_SECONDS`` — and the+# socket won that race in integration: a traceback every ~5.01s while idle, plus a+# disconnect/reconnect each time. Worse, an envelope BLMOVE had already moved to the+# processing list was stranded there until this pod restarted, because the reply never+# reached the client. Identical trap to the one documented at+# ``workers/queue_backend/pg_queue/result_backend.py:152``.+_SOCKET_TIMEOUT_SECONDS = _BLOCK_TIMEOUT_SECONDS + 5++_shutdown = False+++def _processing_list_name() -> str:+ """Per-pod parking list, so one pod never reclaims another's in-flight envelope."""+ return f"{_QUEUE_NAME}:processing:{os.getenv('HOSTNAME') or socket.gethostname()}"+++def _handle_signal(signum: int, _frame: FrameType | None) -> None:+ """Finish the envelope in hand, then stop. Never abandon work mid-flight."""+ global _shutdown+ _shutdown = True+ logger.info("Signal %s received; finishing current log then shutting down", signum)+++def _recover_in_flight(redis_client: Any, processing: str) -> None:+ """Re-queue anything this pod was mid-way through when it last died.++ LMOVE back to the *head* of the source list so recovered envelopes are re-processed+ before newer ones, preserving rough log ordering.+ """+ recovered = 0+ while redis_client.lmove(processing, _QUEUE_NAME, "RIGHT", "LEFT"):+ recovered += 1+ if recovered:+ logger.warning(+ "Recovered %d in-flight log envelope(s) from a previous run of this pod",+ recovered,+ )+++def _dispatch(raw: bytes | str) -> None:+ """Run one envelope through the existing task body.++ Dispatches **by name** so a producer/consumer mismatch fails loudly here rather than+ silently discarding the message.+ """+ envelope = json.loads(raw)+ task_name = envelope.get("task")+ if task_name != LogProcessingTask.TASK_NAME:+ raise ValueError(+ f"Unexpected task {task_name!r} on '{_QUEUE_NAME}'; "+ f"expected {LogProcessingTask.TASK_NAME!r}"+ )+ logs_consumer(**envelope.get("kwargs", {}))+++def run() -> int:+ signal.signal(signal.SIGTERM, _handle_signal)+ signal.signal(signal.SIGINT, _handle_signal)++ # Not ``RedisQueueClient.from_env()``: that hard-codes the 5s socket timeout, which+ # cannot outlive this loop's block. Built directly so the two stay related by+ # construction — see _SOCKET_TIMEOUT_SECONDS.+ redis_client = create_redis_client(+ decode_responses=True,+ socket_timeout=_SOCKET_TIMEOUT_SECONDS,+ )+ processing = _processing_list_name()+ logger.info(+ "Log stream consumer starting: queue='%s' processing='%s'",+ _QUEUE_NAME,+ processing,+ )+ _recover_in_flight(redis_client, processing)++ while not _shutdown:+ try:+ raw = redis_client.blmove(+ _QUEUE_NAME, processing, _BLOCK_TIMEOUT_SECONDS, "LEFT", "RIGHT"+ )+ except Exception:+ # Connection blips must not kill the pod — the next iteration reconnects via+ # the client's own retry. Sleeping is unnecessary: BLMOVE already blocks.+ logger.error("Log stream read failed; retrying", exc_info=True)+ continue++ if raw is None: # timeout, no work — loop so shutdown can be observed+ continue++ try:+ _dispatch(raw)+ except Exception:+ # Match the Celery consumer's posture: a poison envelope is logged and+ # dropped, never retried forever. logs_consumer already swallows its own+ # sink failures, so reaching here means a malformed envelope.+ logger.error("Discarding unprocessable log envelope", exc_info=True)+ finally:+ # Remove exactly one copy, whether it succeeded or was discarded — leaving it+ # parked would have it re-queued on the next restart and replayed forever.+ redis_client.lrem(processing, 1, raw)++ logger.info("Log stream consumer stopped")+ return 0+++if __name__ == "__main__":+ sys.exit(run())diff --git a/workers/queue_backend/pg_queue/metrics.py b/workers/queue_backend/pg_queue/metrics.pyindex 5678b3819..d34c7d885 100644--- a/workers/queue_backend/pg_queue/metrics.py+++ b/workers/queue_backend/pg_queue/metrics.py@@ -252,6 +252,20 @@ class ReaperMetrics(_Exporter):"faults that share pg_reaper_tick_failures_total)",registry=self.registry,)+ self.queue_promoted = Counter(+ "pg_reaper_queue_promoted_total",+ "Due scheduled queue messages promoted to 'ready' (delayed-visibility "+ "delivery for countdown/eta dispatches — the reaper is on the DELIVERY "+ "path for these, not just recovery)",+ registry=self.registry,+ )+ self.queue_promote_failures = Counter(+ "pg_reaper_queue_promote_failures_total",+ "Promotion sweep attempts that raised (delayed messages did not become "+ "claimable this tick; alert on a sustained non-zero rate — the symptom is "+ "silent non-delivery, not an error at the enqueue site)",+ registry=self.registry,+ )self.claim_recovered = Counter("pg_reaper_claim_recovered_total","Orphan orchestration claims recovered (crash-window execution "@@ -268,6 +282,21 @@ class ReaperMetrics(_Exporter):"Orphan-claim recovery attempts that raised (row left for retry)",registry=self.registry,)+ self.undispatched_swept = Counter(+ "pg_reaper_undispatched_swept_total",+ "Executions terminalised because they were created but never dispatched "+ "(no task_id and no queue_message_id past the grace period). These have no "+ "barrier, so barrier recovery cannot see them; a sustained non-zero rate "+ "means requests are dying between create_workflow_execution and dispatch",+ registry=self.registry,+ )+ self.undispatched_sweep_failures = Counter(+ "pg_reaper_undispatched_sweep_failures_total",+ "Undispatched-execution sweep calls that raised (swallowed so the tick "+ "still dispatches schedules; a sustained non-zero rate means PENDING rows "+ "are accumulating unrecovered)",+ registry=self.registry,+ )self.sweep_failures = Counter("pg_reaper_sweep_failures_total","Whole-sweep failures, by swept table (see the reaper fail-streak log)",diff --git a/workers/queue_backend/pg_queue/pg_scheduler.py b/workers/queue_backend/pg_queue/pg_scheduler.pyindex 7ccfe40be..6b87acdaf 100644--- a/workers/queue_backend/pg_queue/pg_scheduler.py+++ b/workers/queue_backend/pg_queue/pg_scheduler.py@@ -56,8 +56,14 @@ SCHEDULER_QUEUE_NAME = "scheduler"class _DueSchedule(NamedTuple):- """One row from the due-schedules scan — names bound to columns at one site- so a future reorder of the SELECT can't silently misassign fields.+ """One row from the due-schedules scan.++ THE FIELD NAMES ARE THE QUERY. They are emitted verbatim as the SELECT's column+ list (see the scan below), so every one MUST be a column of+ ``pg_periodic_schedule`` — renaming a field here rewrites the SQL and needs a+ matching migration in ``backend/pg_queue/models.py``. That is what makes a reorder+ harmless (there is no second list to drift from) and a RENAME dangerous, which is+ the opposite of what an earlier version of this docstring implied."""pipeline_id: uuid.UUID@@ -143,9 +149,8 @@ def dispatch_due_schedules(conn: PgConnection) -> int:base = cur.fetchone()[0]cur.execute(f"""- SELECT pipeline_id, organization_id, workflow_id, pipeline_name,- cron_string, next_run_at- FROM {qualified('pg_periodic_schedule')}+ SELECT {", ".join(f'"{f}"' for f in _DueSchedule._fields)}+ FROM {qualified("pg_periodic_schedule")}WHERE pg_owned AND enabledAND (next_run_at IS NULL OR next_run_at <= %s)""",@@ -230,3 +235,153 @@ def dispatch_due_schedules(conn: PgConnection) -> int:)return fired+++class _DuePeriodicTask(NamedTuple):+ """One row from the generic-periodic due scan (UN-3796).++ Sibling of :class:`_DueSchedule`, and carries the same rule: the field names are+ emitted verbatim as the SELECT's column list, so each MUST be a column of+ ``pg_periodic_task``.++ Note this type says ``org_id`` where its sibling says ``organization_id``. Tempting+ to unify — do not, without a migration. The two back different tables, and renaming+ this one would silently select a column ``pg_periodic_task`` does not have; the+ resulting UndefinedColumn propagates out of the dispatch and takes down the whole+ leader tick, retention sweep and gauge refresh included.+ """++ name: str+ task_name: str+ queue: str+ task_args: list+ task_kwargs: dict+ org_id: str+ cron_string: str+ next_run_at: datetime | None+++def _quiesce_invalid_periodic_cron(conn: PgConnection, row: _DuePeriodicTask) -> None:+ """Disable a generic periodic whose cron won't parse, so it stops being+ re-selected (and re-logging a traceback) every tick. Mirrors+ :func:`_quiesce_invalid_cron` for the pipeline table.+ """+ logger.exception(+ "PG scheduler: invalid cron %r for periodic %r — disabling the row",+ row.cron_string,+ row.name,+ )+ try:+ with conn.cursor() as cur:+ cur.execute(+ f"UPDATE {qualified('pg_periodic_task')} "+ "SET enabled = FALSE WHERE name = %s",+ (row.name,),+ )+ conn.commit()+ except Exception:+ with contextlib.suppress(Exception):+ conn.rollback()+++def dispatch_due_periodic_tasks(conn: PgConnection) -> int:+ """Fire PG-owned, enabled, due **non-pipeline** periodics; return the count fired.++ The Beat-replacement half for everything that isn't a pipeline trigger+ (UN-3796): ``dashboard_metrics.*``, log-history, audit, and anything an operator+ adds. Structurally identical to :func:`dispatch_due_schedules` — leader-gated by+ the caller, DB clock throughout, per-row transaction, enqueue and ``next_run_at``+ advance in ONE transaction so a crash between them cannot double-fire, first+ observation of a freshly-owned row records a baseline without firing.++ The one real difference: each row carries its **own** target queue and its own+ task/args/kwargs, where the pipeline dispatcher rebuilds one fixed argument list+ onto one fixed queue. That is the whole reason for the second table.+ """+ try:+ with conn.cursor() as cur:+ cur.execute("SELECT now()")+ base = cur.fetchone()[0]+ cur.execute(+ f"""+ SELECT {", ".join(f'"{f}"' for f in _DuePeriodicTask._fields)}+ FROM {qualified("pg_periodic_task")}+ WHERE pg_owned AND enabled+ AND (next_run_at IS NULL OR next_run_at <= %s)+ """,+ (base,),+ )+ due = [_DuePeriodicTask(*row) for row in cur.fetchall()]+ conn.commit()+ except Exception:+ with contextlib.suppress(Exception):+ conn.rollback()+ raise++ fired = 0+ for row in due:+ try:+ nxt = compute_next_run(row.cron_string, base)+ except Exception:+ _quiesce_invalid_periodic_cron(conn, row)+ continue++ try:+ if row.next_run_at is None:+ with conn.cursor() as cur:+ cur.execute(+ f"UPDATE {qualified('pg_periodic_task')} "+ "SET next_run_at = %s WHERE name = %s",+ (nxt, row.name),+ )+ conn.commit()+ logger.info(+ "PG scheduler: baselined periodic %r (next_run_at=%s, not fired)",+ row.name,+ nxt,+ )+ continue++ payload = to_payload(+ row.task_name,+ args=list(row.task_args or []),+ kwargs=dict(row.task_kwargs or {}),+ queue=row.queue,+ fairness=None,+ )+ # Enqueue + advance in ONE transaction (see the module docstring).+ with conn.cursor() as cur:+ cur.execute(+ insert_message_sql(),+ (+ row.queue,+ json.dumps(payload),+ row.org_id or "",+ DEFAULT_PRIORITY,+ ),+ )+ cur.execute(+ f"UPDATE {qualified('pg_periodic_task')} "+ "SET last_run_at = %s, next_run_at = %s WHERE name = %s",+ (base, nxt, row.name),+ )+ conn.commit()+ except Exception:+ with contextlib.suppress(Exception):+ conn.rollback()+ logger.exception(+ "PG scheduler: failed to fire periodic %r — leaving for next tick",+ row.name,+ )+ continue++ fired += 1+ logger.info(+ "PG scheduler: fired periodic %r (%s) → %s (next_run_at=%s)",+ row.name,+ row.task_name,+ row.queue,+ nxt,+ )++ return fireddiff --git a/workers/queue_backend/pg_queue/reaper.py b/workers/queue_backend/pg_queue/reaper.pyindex f31a86a93..b43438c44 100644--- a/workers/queue_backend/pg_queue/reaper.py+++ b/workers/queue_backend/pg_queue/reaper.py@@ -70,7 +70,7 @@ from .connection import create_pg_connectionfrom .leader_election import LeaderLease, default_worker_idfrom .liveness import LivenessServer as _BaseLivenessServerfrom .metrics import ReaperMetrics-from .pg_scheduler import dispatch_due_schedules+from .pg_scheduler import dispatch_due_periodic_tasks, dispatch_due_schedulesfrom .recovery import mark_execution_errorfrom .schema import qualified@@ -175,6 +175,46 @@ def reaper_interval_from_env() -> float:return value+#: How long an execution must sit non-terminal with EVERY file already terminal before+#: the safety net finalizes it (``WORKER_PG_STUCK_EXECUTION_RECOVERY_SECONDS``).+#:+#: Ten minutes, and deliberately NOT the barrier stuck-timeout it used to default to.+#: Those answer different questions: the barrier timeout bounds how long a batch may+#: make no progress (hours, because a single file can legitimately take that long),+#: whereas this must outlast the worst-case callback latency after the last file goes+#: terminal (see below — that is what sizes it).+#:+#: **The threshold IS load-bearing — do not lower it casually.** It was not, once: the+#: only guard was all-files-terminal (``internal_views.py``: ``total == 0 or+#: terminal < total → skipped``), and against that guard the value genuinely did not+#: matter. It does now. The selection also requires ``last_file_at < cutoff``, because+#: all-files-terminal alone leaves a live race: between the last file going terminal and+#: the callback finalizing, a HEALTHY execution passes every other check, and a sweep+#: landing there finalizes it first — the status guard then refuses the callback's own+#: write and the notification is lost. This threshold is what sizes that window, so it+#: must exceed the worst-case callback latency after the last file completes. An earlier+#: version of this comment said "seconds" and "no matter how short this is"; acting on+#: that today reopens the race.+#:+#: What remains true is that inheriting the multi-hour barrier timeout is wrong — it+#: answers a different question (how long a batch may make no progress, where one file+#: can legitimately take hours) and it cost a lot: a+#: stranded execution — the common shape being a Celery run whose callback worker was+#: removed by a deploy (its grace is 300s while file-processing gets 7200s) — sat dead+#: for ~2.5 h before anything noticed.+#:+#: A DEFAULT rather than something operators set, because production and on-prem have+#: no operator and no Flipt. A value that only exists as an env override is a value+#: those environments never get, and they are exactly the ones that cannot diagnose a+#: hung execution themselves.+#:+#: Not lowered further because ``total`` counts the file rows that EXIST, not+#: ``execution.total_files`` — while discovery is still creating rows, a brief+#: "all existing rows terminal" moment is possible. Ten minutes clears that comfortably.+#: Comparing against ``total_files`` would close the window properly and allow less.+_DEFAULT_STUCK_RECOVERY_SECONDS = 600++def _positive_duration_from_env(name: str, default: _N, cast: Callable[[str], _N]) -> _N:"""Read a positive duration env var (default on unset; raise on invalid/<=0).@@ -339,6 +379,47 @@ def rearm_expired_claims(conn: PgConnection) -> int:raise+def promote_due_scheduled(conn: PgConnection) -> int:+ """Make deferred queue messages claimable: ``scheduled`` + due -> ``ready``.++ Delayed visibility (UN-3843, Celery ``countdown``/``eta`` parity). A deferred+ row is enqueued ``state='scheduled'`` with a future ``available_at`` and is+ **absent from the claim's partial index**, so consumers cannot see it and it+ costs the hot claim path nothing while it waits. This sweep is the only thing+ that promotes it, which makes the reaper part of the delivery path for delayed+ messages — not just the recovery path. It is already the mandatory singleton for+ crash redelivery, so no new deployment dependency, but a queue that uses delays+ inherits its liveness alert.++ Consequence to hold onto: delivery is **"not before ``available_at``"**, never+ early, with granularity of the reaper tick (default 5s) — not exact ETA. Celery's+ countdown is likewise approximate, and every consumer of this (staggered sends,+ retry backoff) needs a floor rather than an instant.++ Cheap and bounded: ``pg_queue_message_scheduled_idx`` (partial, ``scheduled``+ only, keyed on ``available_at``) means a backlog of far-future rows costs one+ index seek rather than a scan. Queue-agnostic — one statement promotes every due+ row across all queues. Idempotent (a promoted row leaves the predicate); rolls+ back on error. Runs every **leader** tick alongside+ :func:`rearm_expired_claims`; see :meth:`PgReaper.tick`.+ """+ ready = QueueMessageState.READY.value+ scheduled = QueueMessageState.SCHEDULED.value+ try:+ with conn.cursor() as cur:+ cur.execute(+ f"UPDATE {qualified('pg_queue_message')} "+ f"SET state = '{ready}' "+ f"WHERE state = '{scheduled}' AND available_at <= now()"+ )+ promoted = cur.rowcount+ conn.commit()+ return promoted+ except Exception:+ _rollback_after_sweep_failure(conn, "pg_queue_message")+ raise++def _execution_status(api_client: InternalAPIClient, execution_id: str, organization_id: str) -> str | object | None:@@ -1038,15 +1119,13 @@ class PgReaper:)if self._dedup_retention <= 0:raise ValueError("dedup_retention_seconds must be positive")- # Safety-net recovery of PG executions stranded non-terminal after all files+ # Safety-net recovery of executions stranded non-terminal after all files# completed (barrier gone → invisible to the PG-table sweeps). Enablement +- # kill-switch semantics live in stuck_recovery_enabled_from_env(); the stuck- # window (below) defaults to the barrier stuck-timeout so it never races a- # legitimately long execution.+ # kill-switch semantics live in stuck_recovery_enabled_from_env().self._stuck_recovery_enabled = stuck_recovery_enabled_from_env()self._stuck_recovery_seconds = _positive_duration_from_env("WORKER_PG_STUCK_EXECUTION_RECOVERY_SECONDS",- self._stuck_timeout_seconds,+ _DEFAULT_STUCK_RECOVERY_SECONDS,int,)# None → "never swept", so the first leader tick sweeps immediately; set to@@ -1186,6 +1265,31 @@ class PgReaper:)self._discard_owned_sweep_conn()raise+ # Delayed-visibility delivery (UN-3843): promote due 'scheduled' rows so+ # consumers can claim them. Placed with the re-arm sweep because it shares+ # its cadence requirement — this is the DELIVERY path for delayed messages,+ # so a slower interval would directly add latency to every countdown/eta+ # dispatch. Same failure posture as the re-arm above (dedicated counter,+ # re-raise, discard the conn): a stalled promotion sweep means delayed+ # messages silently never fire, which must not be swallowed.+ try:+ promoted = promote_due_scheduled(self._get_sweep_conn())+ if promoted:+ self._metrics.queue_promoted.inc(promoted)+ logger.info(+ "Reaper: promoted %s due scheduled queue message(s) to 'ready' "+ "(delayed-visibility delivery)",+ promoted,+ )+ except Exception:+ self._metrics.queue_promote_failures.inc()+ logger.exception(+ "Reaper: promotion sweep failed — delayed (countdown/eta) queue "+ "messages will not become claimable this tick "+ "(see pg_reaper_queue_promote_failures_total)"+ )+ self._discard_owned_sweep_conn()+ raise# Orchestrator's second job: fire due PG-owned schedules (Beat# replacement). Ordered AFTER recovery so this cycle's recovery has# already completed before any scheduler error can propagate (the except@@ -1196,6 +1300,17 @@ class PgReaper:except Exception:self._discard_owned_sweep_conn()raise+ # ...and the non-pipeline periodics (UN-3796): dashboard_metrics.*,+ # log-history, audit, anything an operator adds. Separate call because each+ # row carries its own task/args/queue rather than the pipeline trigger's one+ # fixed shape; same leader gating, same dark-by-default posture (nothing+ # fires until a row is pg_owned). Ordered after the pipeline dispatch so a+ # fault here cannot stop pipelines, which are the customer-visible ones.+ try:+ dispatch_due_periodic_tasks(self._get_sweep_conn())+ except Exception:+ self._discard_owned_sweep_conn()+ raise# Orchestrator's third job: retention cleanup (cadence-gated, so it does# NOT run every tick). Last so a sweep error can't skip recovery/schedules.self._maybe_sweep()@@ -1249,8 +1364,59 @@ class PgReaper:dedup,claims,)+ self._sweep_undispatched_executions()self._maybe_recover_stuck_executions()+ def _sweep_undispatched_executions(self) -> None:+ """Terminalise executions created but never dispatched (backend-side).++ The gap this closes: an abort between ``create_workflow_execution`` and+ ``execute_workflow_async`` commits a PENDING row that was never queued. No+ queue message and no barrier are ever created, so+ :func:`recover_expired_barriers` — which scans ``pg_barrier_state`` — cannot+ see it, and PENDING is not terminal, so nothing else resolves it either.+ Observed as 967 orphans from one load test whose API tier shed 71% of traffic.++ Delegated to the backend rather than done here: this reaper deliberately never+ touches backend tables (it reads/writes execution state through the internal+ API), and the sweep also needs the rate limiter.+ The reaper contributes what it uniquely has — leader election, so exactly one+ instance sweeps — while the backend owns the logic.++ Cadence-gated with the retention sweeps: these rows are already older than the+ grace period, so there is nothing to gain from running it every tick.++ **Swallowed on failure**, unlike barrier recovery. This cleans up work that is+ already dead; a fault here must not discard the connection or abort the tick+ and thereby defer schedule dispatch, which serves live traffic.+ """+ try:+ response = self._get_api_client().sweep_undispatched_executions()+ except Exception:+ self._metrics.undispatched_sweep_failures.inc()+ logger.warning(+ "Reaper: undispatched-execution sweep failed; retrying next sweep",+ exc_info=True,+ )+ return+ # sweep_undispatched_executions() is typed `-> dict[str, Any]` and returns the+ # parsed body verbatim — {"swept": N}, no {"data": ...} envelope. Reading+ # `.data` off a plain dict yields None, so this counter was always 0: the+ # metric never incremented and the "terminalised N" line never fired, leaving+ # the 967-orphan condition this sweep exists to detect completely invisible.+ # The sibling recover_stuck_pg_executions() hits the same flat-body shape and+ # documents it (execution_client.py:352-358); it wraps there instead, which is+ # why that one works. Read the dict directly here rather than add a second+ # wrapping convention.+ swept = (response if isinstance(response, dict) else {}).get("swept", 0)+ if swept:+ self._metrics.undispatched_swept.inc(swept)+ logger.info(+ "Reaper: terminalised %s undispatched execution(s) "+ "(created but never queued)",+ swept,+ )+def _maybe_recover_stuck_executions(self) -> None:"""Opt-in safety-net: finalize PG executions stranded non-terminal after allfiles completed.diff --git a/workers/queue_backend/pg_queue/schema.py b/workers/queue_backend/pg_queue/schema.pyindex 92273ae02..98976efd7 100644--- a/workers/queue_backend/pg_queue/schema.py+++ b/workers/queue_backend/pg_queue/schema.py@@ -53,6 +53,7 @@ QUEUE_TABLES = frozenset("pg_orchestration_claim","pg_orchestrator_lock","pg_periodic_schedule",+ "pg_periodic_task",})diff --git a/workers/run-worker-docker.sh b/workers/run-worker-docker.shindex be5aa8678..05d2b9398 100755--- a/workers/run-worker-docker.sh+++ b/workers/run-worker-docker.sh@@ -536,6 +536,19 @@ run_pg_consumer() {exec "$PG_QUEUE_PYTHON_BIN" -m pg_queue_consumer}+run_log_stream_consumer() {+ ensure_pg_interpreter+ export WORKER_NAME="${WORKER_NAME:-log-stream-consumer}"++ # Named for its transport, not the `pg-` family: this drains a Redis list, not+ # pg_queue_message. It is nonetheless the flag-on replacement for the Celery+ # worker-log-consumer, which is why it lives beside the PG components here.+ print_status $GREEN "Starting log stream consumer (Redis transport)..."+ print_status $BLUE "Queue: ${LOG_STREAM_QUEUE_NAME:-log_stream_queue}"++ exec "$PG_QUEUE_PYTHON_BIN" -m log_consumer.redis_stream_consumer+}+run_pg_reaper() {ensure_pg_interpreterexport WORKER_NAME="${WORKER_NAME:-pg-reaper}"@@ -567,6 +580,9 @@ case "${1:-}" inpg-queue-reaper|pg-reaper|reaper)run_pg_reaper;;+ log-stream-consumer)+ run_log_stream_consumer+ ;;pg-*)# Obviously-PG-intended but unrecognized (e.g. a typo'd command) — fail# loudly instead of silently coercing it into a default Celery worker.diff --git a/workers/run-worker.sh b/workers/run-worker.shindex 0ec399dee..5c37466bf 100755--- a/workers/run-worker.sh+++ b/workers/run-worker.sh@@ -57,6 +57,7 @@ readonly PG_ROLE_FILEPROC="pg-fileproc"readonly PG_ROLE_CALLBACK="pg-callback"readonly PG_ROLE_SCHEDULER="pg-scheduler"readonly PG_ROLE_EXECUTOR="pg-executor"+readonly PG_ROLE_METRICS="pg-metrics"declare -rA PG_CONSUMER_ROLES=(["$PG_ROLE_ORCH_API"]="api_deployment;celery_api_deployments"["$PG_ROLE_ORCH_GENERAL"]="general;celery"@@ -70,6 +71,14 @@ declare -rA PG_CONSUMER_ROLES=(# the result to pg_task_result for the blocking caller. Queues mirror the# Celery executor's CELERY_QUEUES_EXECUTOR.["$PG_ROLE_EXECUTOR"]="executor;celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table"+ # Runs the dashboard_metrics.* periodics fired by the PG scheduler tick,+ # replacing the Celery 'workerMetrics' (-Q dashboard_metric_events). Its own+ # role rather than a queue bolted onto pg-scheduler: the consumer's health+ # heartbeat freezes for the duration of a task, so HEALTH_STALE is an upper+ # bound on one task's wall clock. The aggregation runs for minutes; sharing+ # pg-scheduler's tight 240s bound would restart that pod and take in-flight+ # pipeline triggers down with it. Sized in docker-compose.yaml.+ ["$PG_ROLE_METRICS"]="scheduler;dashboard_metric_events")declare -rA PG_QUEUE_MEMBERS=(["$PG_QUEUE_CONSUMER_TYPE"]=1@@ -80,6 +89,7 @@ declare -rA PG_QUEUE_MEMBERS=(["$PG_ROLE_CALLBACK"]=1["$PG_ROLE_SCHEDULER"]=1["$PG_ROLE_EXECUTOR"]=1+ ["$PG_ROLE_METRICS"]=1)# The Celery transport set: every worker EXCEPT the PG-queue members — the# *complement* of the 'pg-queue' set, so the two transports' logs can be tailed@@ -119,6 +129,7 @@ declare -A WORKERS=(["$PG_ROLE_CALLBACK"]="$PG_ROLE_CALLBACK"["$PG_ROLE_SCHEDULER"]="$PG_ROLE_SCHEDULER"["$PG_ROLE_EXECUTOR"]="$PG_ROLE_EXECUTOR"+ ["$PG_ROLE_METRICS"]="$PG_ROLE_METRICS"# PG Queue reaper — leader-elected recovery loop (barrier-orphan sweep)["reaper"]="$PG_QUEUE_REAPER_TYPE"["pg-queue-reaper"]="$PG_QUEUE_REAPER_TYPE"diff --git a/workers/scheduler/dashboard_metrics_tasks.py b/workers/scheduler/dashboard_metrics_tasks.pynew file mode 100644index 000000000..44bbe5044--- /dev/null+++ b/workers/scheduler/dashboard_metrics_tasks.py@@ -0,0 +1,126 @@+"""Thin PG-side tasks for the dashboard-metrics periodics (UN-3796).++These replace ``workerMetrics`` (``celery -A backend worker -Q dashboard_metric_events``)+once the matching schedules are adopted by the PG scheduler. They do **no** work+themselves: each POSTs to a backend internal endpoint that calls the real Django+function, mirroring ``log_consumer/process_log_history.py`` and+``process_notification_buffer.py``. The workers image has no Django, so the ORM-heavy+aggregation cannot run here.++**Registered under the exact Beat task names.** That is deliberate: the mirror copies a+Beat row verbatim (task, queue, args, kwargs), so matching names mean no remap table and+``--release`` stays a true inverse. The same name therefore exists in two registries —+the backend image (the Django implementation, consumed by Celery) and this one (the HTTP+proxy, consumed by ``worker-pg-metrics``). They are separate processes with separate+registries and neither imports the other; the name is a logical contract, not a symbol+clash. A reader who assumes otherwise will be very confused, hence this paragraph.++Redelivery is safe without extra guards: the backend's upserts are+``INSERT … ON CONFLICT DO UPDATE SET`` (overwrite with recomputed values, not+increment) and the cleanups are ``DELETE … WHERE ts < cutoff``, so a double-run costs+duplicate DB work, never wrong numbers.+"""++from __future__ import annotations++import os+from typing import Any++import httpx+from queue_backend import worker_task+from shared.infrastructure.logging import WorkerLogger++logger = WorkerLogger.get_logger(__name__)++# Sits just above gunicorn's --timeout 600, which is the ceiling that actually applies+# here (the task's Celery soft_time_limit=600 / time_limit=660 govern the Celery path,+# where a Celery worker runs the body in-process — not this one). Above it so a long+# aggregation surfaces the SERVER's error; a shorter client timeout would abort first+# and read as a network fault while the server kept working.+DEFAULT_HTTP_TIMEOUT_SECONDS = 630.0++_AGGREGATE_PATH = "v1/dashboard-metrics/aggregate/"+_CLEANUP_HOURLY_PATH = "v1/dashboard-metrics/cleanup/hourly/"+_CLEANUP_DAILY_PATH = "v1/dashboard-metrics/cleanup/daily/"+++def _call_internal(+ path: str,+ *,+ method: str = "POST",+ body: dict[str, Any] | None = None,+ timeout: float = DEFAULT_HTTP_TIMEOUT_SECONDS,+) -> dict[str, Any]:+ """Call a backend internal endpoint and return its decoded body.++ Single seam so tests have one thing to patch.++ **Raises** on any failure, unlike ``process_log_history.py`` which returns False.+ That script is driven by a bash loop with no other channel; this runs inside a PG+ consumer, where raising is what marks the message failed and gets it logged loudly+ (with ``MAX_ATTEMPTS=1`` it is then dropped rather than retried — the next cron tick+ supersedes it).++ Never sends ``X-Organization-ID``: these are global aggregations, and the middleware+ would otherwise scope every ORM read in them to a single tenant.+ """+ base_url = os.getenv("INTERNAL_API_BASE_URL")+ api_key = os.getenv("INTERNAL_SERVICE_API_KEY")+ if not base_url:+ raise RuntimeError("INTERNAL_API_BASE_URL environment variable not set")+ if not api_key:+ raise RuntimeError("INTERNAL_SERVICE_API_KEY environment variable not set")++ url = f"{base_url.rstrip('/')}/{path}"+ # Transport-level retries only — these re-establish a connection that never+ # delivered the request. They do NOT re-send after the server received it, which+ # matters: a retry on a slow-but-live aggregation would run it twice concurrently.+ transport = httpx.HTTPTransport(retries=3)+ with httpx.Client(transport=transport) as client:+ response = client.request(+ method,+ url,+ headers={"Authorization": f"Bearer {api_key}"},+ json=body,+ timeout=timeout,+ )+ if response.status_code != 200:+ raise RuntimeError(+ f"{method} {path} failed: HTTP {response.status_code} {response.text[:500]}"+ )+ return response.json()+++def _log_if_skipped(name: str, result: dict[str, Any]) -> None:+ """Surface a lock-held no-op.++ The backend returns success with ``skipped=True`` when the Redis lock is held. That+ is correct behaviour, but left at INFO a permanently leaked lock looks like 96+ successful runs a day that did nothing.+ """+ if result.get("skipped"):+ logger.warning(+ "%s did no work: %s", name, result.get("reason", "reported skipped=True")+ )+++@worker_task(name="dashboard_metrics.aggregate_from_sources")+def dashboard_metrics_aggregate() -> dict[str, Any]:+ """Aggregate source tables into the hourly/daily/monthly metrics tables."""+ result = _call_internal(_AGGREGATE_PATH)+ _log_if_skipped("dashboard_metrics.aggregate_from_sources", result)+ return result+++@worker_task(name="dashboard_metrics.cleanup_hourly_data")+def dashboard_metrics_cleanup_hourly(retention_days: int | None = None) -> dict[str, Any]:+ """Delete hourly metrics older than the retention window."""+ body = {"retention_days": retention_days} if retention_days is not None else None+ return _call_internal(_CLEANUP_HOURLY_PATH, body=body)+++@worker_task(name="dashboard_metrics.cleanup_daily_data")+def dashboard_metrics_cleanup_daily(retention_days: int | None = None) -> dict[str, Any]:+ """Delete daily metrics older than the retention window."""+ body = {"retention_days": retention_days} if retention_days is not None else None+ return _call_internal(_CLEANUP_DAILY_PATH, body=body)diff --git a/workers/scheduler/tasks.py b/workers/scheduler/tasks.pyindex 3c98fda9b..b30742c45 100644--- a/workers/scheduler/tasks.py+++ b/workers/scheduler/tasks.py@@ -9,6 +9,21 @@ from typing import Anyfrom queue_backend import FairnessKey, QueueBackend, dispatch, worker_taskfrom queue_backend.fairness import WorkloadType++# Register the dashboard-metrics proxy tasks on this worker type (UN-3796). Imported+# purely for the side effect.+#+# ABSOLUTE package import, and it must stay that way — this module is reached by TWO+# different mechanisms and only this form survives both:+# 1. worker.py loads /app/scheduler/tasks.py BY PATH with the worker dir on sys.path.+# No parent package is set, so `from . import ...` cannot resolve.+# 2. general/tasks.py does `from scheduler.tasks import execute_pipeline_task_v2`,+# importing it as a package module. Then /app/scheduler is NOT on sys.path, so a+# bare `import dashboard_metrics_tasks` raises ModuleNotFoundError and crash-loops+# worker-general (this happened — the bare form shipped and broke the general+# worker at flag-off, where PG is not even involved).+# `/app` is on PYTHONPATH (run-worker-docker.sh), so `scheduler.` resolves under both.+from scheduler import dashboard_metrics_tasks # noqa: F401, E402 (side-effect import)from shared.enums.status_enums import PipelineStatusfrom shared.enums.worker_enums import QueueNamefrom shared.infrastructure.config import WorkerConfigdiff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.pyindex a81287ccf..bba4af529 100644--- a/workers/shared/api/internal_client.py+++ b/workers/shared/api/internal_client.py@@ -1445,6 +1445,21 @@ class InternalAPIClient(CachedAPIClientMixin):"""Make GET request."""return self.base_client.get(endpoint, params, organization_id)+ def sweep_undispatched_executions(self) -> dict[str, Any]:+ """Ask the backend to terminalise executions that were never dispatched.++ Closes a gap no worker can see: an abort between ``create_workflow_execution``+ and ``execute_workflow_async`` commits a PENDING execution row without ever+ creating a queue message or a barrier, so the reaper's barrier scan cannot+ find it and PENDING — a non-terminal state — is never resolved.++ Backend-side because the sweep touches ``WorkflowExecution``, the API-deployment+ rate limiter; the reaper only supplies leader+ election so exactly one instance runs it. Org-agnostic: the sweep spans all+ organizations, so no ``organization_id`` is sent.+ """+ return self.post("v1/execution-logs/sweep-undispatched-executions/", data={})+def post(self, endpoint: str, data: dict[str, Any], organization_id: str | None = None) -> dict[str, Any]:diff --git a/workers/tests/test_dashboard_metrics_tasks.py b/workers/tests/test_dashboard_metrics_tasks.pynew file mode 100644index 000000000..ce8ff853a--- /dev/null+++ b/workers/tests/test_dashboard_metrics_tasks.py@@ -0,0 +1,147 @@+"""Tests for the thin dashboard-metrics proxy tasks (UN-3796).++The tasks themselves do nothing but call a backend internal endpoint, so what is worth+pinning is the contract around that call: the registered names (a mismatch means the PG+consumer drops the message as an unknown task — the failure mode with no error at the+enqueue site), the request shape, and the failure posture.+"""++from __future__ import annotations++import sys+from pathlib import Path+from unittest.mock import MagicMock, patch++import pytest++# Import via the PACKAGE, matching `scheduler/tasks.py`'s+# `from scheduler import dashboard_metrics_tasks` (OSS 6397dd578) — which is the form+# BOTH runtime mechanisms converge on: worker.py's by-path load of tasks.py still+# resolves that import against /app, so `scheduler.dashboard_metrics_tasks` is the+# single module object either way.+#+# The bare `import dashboard_metrics_tasks` used here previously mirrored only the+# by-path load, and once tasks.py started importing the package form the two coexisted+# as SEPARATE module objects with separate Celery registrations. Patching one left the+# other live, so a test that believed it had mocked the HTTP client made a REAL request+# and failed on DNS — but only when some earlier test in the run had already imported+# `scheduler.tasks`, which is why it looked like flakiness rather than a wiring bug.+_WORKERS_ROOT = Path(__file__).resolve().parent.parent+if str(_WORKERS_ROOT) not in sys.path:+ sys.path.insert(0, str(_WORKERS_ROOT))++from scheduler import dashboard_metrics_tasks as dmt # noqa: E402++_ENV = {+ "INTERNAL_API_BASE_URL": "http://backend:8000/internal",+ "INTERNAL_SERVICE_API_KEY": "test-key",+}++[email protected](autouse=True)+def _env(monkeypatch):+ for k, v in _ENV.items():+ monkeypatch.setenv(k, v)+++class TestRegistration:+ """The names must match the Beat rows exactly, or the mirror's verbatim copy+ produces a message no consumer can resolve — silently dropped as poison.+ """++ @pytest.mark.parametrize(+ "name,func",+ [+ ("dashboard_metrics.aggregate_from_sources", "dashboard_metrics_aggregate"),+ ("dashboard_metrics.cleanup_hourly_data", "dashboard_metrics_cleanup_hourly"),+ ("dashboard_metrics.cleanup_daily_data", "dashboard_metrics_cleanup_daily"),+ ],+ )+ def test_task_is_registered_under_the_beat_name(self, name, func):+ assert getattr(dmt, func).name == name+++class TestCallContract:+ def test_aggregate_posts_to_the_aggregate_endpoint(self):+ with patch.object(dmt, "_call_internal", return_value={"success": True}) as call:+ dmt.dashboard_metrics_aggregate()+ assert call.call_args[0][0] == "v1/dashboard-metrics/aggregate/"++ @pytest.mark.parametrize(+ "func,path",+ [+ ("dashboard_metrics_cleanup_hourly", "v1/dashboard-metrics/cleanup/hourly/"),+ ("dashboard_metrics_cleanup_daily", "v1/dashboard-metrics/cleanup/daily/"),+ ],+ )+ def test_cleanup_passes_retention_through(self, func, path):+ with patch.object(dmt, "_call_internal", return_value={"deleted": 1}) as call:+ getattr(dmt, func)(retention_days=45)+ assert call.call_args[0][0] == path+ assert call.call_args.kwargs["body"] == {"retention_days": 45}++ def test_cleanup_omits_body_when_no_retention_given(self):+ # The backend then applies the same default the Beat kwargs carry, so an+ # unspecified call matches the Celery path rather than inventing a value here.+ with patch.object(dmt, "_call_internal", return_value={}) as call:+ dmt.dashboard_metrics_cleanup_hourly()+ assert call.call_args.kwargs["body"] is None++ def test_lock_held_result_is_surfaced_not_swallowed(self, caplog):+ # A permanently leaked lock otherwise looks like 96 successful no-op runs a day.+ with patch.object(+ dmt,+ "_call_internal",+ return_value={"success": True, "skipped": True, "reason": "lock_held"},+ ):+ result = dmt.dashboard_metrics_aggregate()+ assert result["skipped"] is True+++class TestInternalCall:+ def _response(self, status_code=200, payload=None):+ r = MagicMock()+ r.status_code = status_code+ r.json.return_value = payload if payload is not None else {"ok": True}+ r.text = "boom"+ return r++ def test_sends_bearer_auth_and_never_an_org_header(self):+ # X-Organization-ID would make the middleware scope every ORM read in these+ # global aggregations to one tenant.+ with patch.object(dmt.httpx, "Client") as client_cls:+ client = client_cls.return_value.__enter__.return_value+ client.request.return_value = self._response()+ dmt._call_internal("v1/x/")+ headers = client.request.call_args.kwargs["headers"]+ assert headers == {"Authorization": "Bearer test-key"}+ assert not any(h.lower() == "x-organization-id" for h in headers)++ def test_timeout_outlasts_the_server_side_ceiling(self):+ # The ceiling that matters is gunicorn's --timeout 600, not the task's Celery+ # time_limit=660 (no Celery worker runs it on this path). The client must sit+ # ABOVE 600 so a long run surfaces the server's error rather than our own+ # timeout — which would look like a network fault and hide the real cause.+ with patch.object(dmt.httpx, "Client") as client_cls:+ client = client_cls.return_value.__enter__.return_value+ client.request.return_value = self._response()+ dmt._call_internal("v1/x/")+ assert client.request.call_args.kwargs["timeout"] > 600++ def test_non_200_raises(self):+ with patch.object(dmt.httpx, "Client") as client_cls:+ client = client_cls.return_value.__enter__.return_value+ client.request.return_value = self._response(status_code=500)+ with pytest.raises(RuntimeError, match="HTTP 500"):+ dmt._call_internal("v1/x/")++ @pytest.mark.parametrize(+ "missing", ["INTERNAL_API_BASE_URL", "INTERNAL_SERVICE_API_KEY"]+ )+ def test_missing_config_raises_rather_than_returning_falsy(self, monkeypatch, missing):+ # Deliberately different from process_log_history.py, which returns False: that+ # runs under a bash loop with no other channel. Here raising is what marks the+ # message failed and gets it logged.+ monkeypatch.delenv(missing)+ with pytest.raises(RuntimeError, match=missing):+ dmt._call_internal("v1/x/")diff --git a/workers/tests/test_log_stream_consumer.py b/workers/tests/test_log_stream_consumer.pynew file mode 100644index 000000000..8c3f3091e--- /dev/null+++ b/workers/tests/test_log_stream_consumer.py@@ -0,0 +1,212 @@+"""Tests for the Redis log-stream consumer's delivery semantics (UN-3755).++The consumer replaces a Celery worker running with ``task_acks_late=True``, where a+crash mid-task means redelivery. A bare ``BLPOP`` would have quietly made crashes lossy,+so the loop parks each envelope on a per-pod processing list and removes it only after+the handler returns. These pin that contract — it is the part that is easy to regress+into "logs vanish when a pod restarts" without any test noticing.++The module is loaded over faked worker-framework imports so the test needs neither a+Celery app nor a live Redis.+"""++from __future__ import annotations++import importlib.util+import json+import sys+import types+from pathlib import Path+from unittest.mock import MagicMock, patch++import pytest++_MODULE = (+ Path(__file__).resolve().parent.parent / "log_consumer" / "redis_stream_consumer.py"+)+++def _load(monkeypatch):+ """Import the consumer with its framework + task imports stubbed out."""+ monkeypatch.setenv("LOG_STREAM_QUEUE_NAME", "log_stream_queue")+ monkeypatch.setenv("HOSTNAME", "pod-abc")++ def _mod(name, **attrs):+ m = types.ModuleType(name)+ for k, v in attrs.items():+ setattr(m, k, v)+ return m++ logs_consumer = MagicMock(name="logs_consumer")+ stubs = {+ "shared": _mod("shared"),+ "shared.enums": _mod("shared.enums"),+ "shared.enums.worker_enums": _mod(+ "shared.enums.worker_enums",+ WorkerType=types.SimpleNamespace(LOG_CONSUMER="log_consumer"),+ ),+ "shared.infrastructure": _mod("shared.infrastructure"),+ "shared.infrastructure.config": _mod("shared.infrastructure.config"),+ "shared.infrastructure.config.builder": _mod(+ "shared.infrastructure.config.builder",+ WorkerBuilder=types.SimpleNamespace(+ build_celery_app=lambda _t: (MagicMock(), MagicMock())+ ),+ ),+ "shared.infrastructure.logging": _mod(+ "shared.infrastructure.logging",+ WorkerLogger=types.SimpleNamespace(setup=lambda _t: MagicMock()),+ ),+ "log_consumer": _mod("log_consumer"),+ "log_consumer.tasks": _mod("log_consumer.tasks", logs_consumer=logs_consumer),+ }+ for name, mod in stubs.items():+ monkeypatch.setitem(sys.modules, name, mod)++ spec = importlib.util.spec_from_file_location(+ "log_consumer.redis_stream_consumer", _MODULE+ )+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ mod._test_logs_consumer = logs_consumer+ return mod+++def consumer(monkeypatch):+ return _load(monkeypatch)+++def _envelope(task="logs_consumer", **kwargs):+ return json.dumps({"task": task, "kwargs": kwargs or {"event": "logs:c", "room": "c"}})+++class TestDispatch:+ def test_runs_the_existing_task_body_with_the_envelope_kwargs(self, consumer):+ consumer._dispatch(_envelope(event="logs:c1", user_session_id="c1"))+ consumer._test_logs_consumer.assert_called_once_with(+ event="logs:c1", user_session_id="c1"+ )++ def test_rejects_an_unexpected_task_name_loudly(self, consumer):+ # Dispatching blind would run the log handler on a foreign payload; raising here+ # surfaces a producer/consumer mismatch instead of corrupting the stream.+ with pytest.raises(ValueError, match="Unexpected task"):+ consumer._dispatch(_envelope(task="something_else"))+ consumer._test_logs_consumer.assert_not_called()+++class TestAtLeastOnceDelivery:+ def test_processing_list_is_scoped_to_this_pod(self, consumer):+ # A shared list would let one pod reclaim another's in-flight envelope and+ # replay it while the owner is still working on it.+ assert consumer._processing_list_name() == "log_stream_queue:processing:pod-abc"++ def test_startup_requeues_what_the_previous_run_left_in_flight(self, consumer):+ redis = MagicMock()+ redis.lmove.side_effect = [b"a", b"b", None]+ consumer._recover_in_flight(redis, "proc")+ assert redis.lmove.call_count == 3+ # Back to the HEAD of the source list, so recovered logs precede newer ones.+ assert redis.lmove.call_args_list[0][0] == ("proc", "log_stream_queue", "RIGHT", "LEFT")++ def _one_shot_redis(self, consumer, raw):+ """A redis mock that yields exactly one envelope, then ends the loop.++ ``lmove`` must return None or startup recovery spins forever — a real Redis+ returns nil on an empty list, but a bare MagicMock is truthy.+ """+ redis = MagicMock()+ redis.lmove.return_value = None++ def _blmove(*_a, **_k):+ if redis.blmove.call_count == 1:+ return raw+ consumer._shutdown = True+ return None++ redis.blmove.side_effect = _blmove+ return redis++ def test_envelope_is_removed_only_after_the_handler_returns(self, consumer):+ raw = _envelope()+ redis = self._one_shot_redis(consumer, raw)+ order = []+ redis.lrem.side_effect = lambda *a: order.append("lrem")+ consumer._test_logs_consumer.side_effect = lambda **_: order.append("handled")++ with patch.object(consumer, "create_redis_client", return_value=redis):+ consumer.run()++ # Order is the whole point: lrem before the handler would lose the envelope on+ # a crash, which is exactly the acks_late behaviour this replaces.+ assert order == ["handled", "lrem"]+ redis.lrem.assert_called_once_with("log_stream_queue:processing:pod-abc", 1, raw)++ def test_a_poison_envelope_is_dropped_not_replayed_forever(self, consumer):+ raw = b"not-json"+ redis = self._one_shot_redis(consumer, raw)+ with patch.object(consumer, "create_redis_client", return_value=redis):+ consumer.run()++ # Still removed from the processing list — otherwise startup recovery would+ # re-queue it on every restart and the loop would never drain.+ redis.lrem.assert_called_once_with("log_stream_queue:processing:pod-abc", 1, raw)+++class TestSocketTimeoutOutlivesTheBlock:+ """redis-py applies ``socket_timeout`` to the BLMOVE read itself.++ Shipped equal to the block (both 5s) and the socket won the race in integration:+ ``redis.exceptions.TimeoutError: Timeout reading from socket`` every ~5.01s while+ idle, each one tearing down the connection. The quiet part is worse — BLMOVE is+ atomic server-side, so an envelope could be moved onto the processing list and its+ reply then lost with the socket, stranding that log until the pod restarted.+ """++ def test_socket_timeout_strictly_exceeds_the_block_timeout(self, consumer):+ assert consumer._SOCKET_TIMEOUT_SECONDS > consumer._BLOCK_TIMEOUT_SECONDS++ def test_the_margin_holds_when_the_block_is_tuned_up(self, monkeypatch):+ # The two must stay related by construction, not by both happening to be+ # defaults — a deployment raising the block alone would resurrect the bug.+ monkeypatch.setenv("LOG_STREAM_BLOCK_TIMEOUT", "45")+ mod = _load(monkeypatch)+ assert mod._BLOCK_TIMEOUT_SECONDS == 45+ assert mod._SOCKET_TIMEOUT_SECONDS > 45++ def test_the_client_is_actually_built_with_that_timeout(self, consumer):+ """The constant is inert unless it reaches ``create_redis_client``."""+ redis = MagicMock()+ redis.lmove.return_value = None++ def _blmove(*_a, **_k):+ consumer._shutdown = True+ return None++ redis.blmove.side_effect = _blmove++ with patch.object(+ consumer, "create_redis_client", return_value=redis+ ) as factory:+ consumer.run()++ assert factory.call_args.kwargs["socket_timeout"] == (+ consumer._SOCKET_TIMEOUT_SECONDS+ )++ def test_blmove_is_called_with_the_block_timeout(self, consumer):+ """Pins the other half of the pair the invariant is about."""+ redis = MagicMock()+ redis.lmove.return_value = None++ def _blmove(*_a, **_k):+ consumer._shutdown = True+ return None++ redis.blmove.side_effect = _blmove++ with patch.object(consumer, "create_redis_client", return_value=redis):+ consumer.run()++ assert redis.blmove.call_args[0][2] == consumer._BLOCK_TIMEOUT_SECONDSdiff --git a/workers/tests/test_log_stream_transport.py b/workers/tests/test_log_stream_transport.pynew file mode 100644index 000000000..34d8cf6f1--- /dev/null+++ b/workers/tests/test_log_stream_transport.py@@ -0,0 +1,111 @@+"""Tests for the log-streaming transport branch (UN-3755).++``LogPublisher.publish`` is the hop that lets the log consumer stop being a Celery+worker. What matters is that the flag-off path is untouched (it is what staging and+production run), that the flag-on envelope carries the task **name** so the consumer can+dispatch it, and that a logging fault can never break an execution.+"""++from __future__ import annotations++import json+from unittest.mock import MagicMock, patch++import pytest++from unstract.core.constants import LogProcessingTask+from unstract.core.pubsub_helper import LogPublisher++_PAYLOAD = {"type": "LOG", "level": "INFO", "log": "hello", "timestamp": 1.0}+++def redis_client():+ client = MagicMock()+ client.llen.return_value = 0+ with patch.object(LogPublisher, "_get_redis_client", return_value=client):+ yield client+++def kombu():+ with patch.object(LogPublisher, "kombu_conn") as conn:+ yield conn.Producer.return_value.__enter__.return_value+++class TestFlagOff:+ """Default transport must remain exactly what it is on main."""++ def test_publishes_to_amqp_and_never_touches_the_redis_stream(+ self, monkeypatch, kombu, redis_client+ ):+ monkeypatch.delenv("LOG_TRANSPORT", raising=False)+ assert LogPublisher.publish("chan-1", _PAYLOAD) is True++ kombu.publish.assert_called_once()+ kwargs = kombu.publish.call_args.kwargs+ assert kwargs["routing_key"] == LogProcessingTask.QUEUE_NAME+ assert kwargs["headers"] == {"task": LogProcessingTask.TASK_NAME}+ redis_client.rpush.assert_not_called()++ def test_an_unrelated_transport_value_still_uses_celery(+ self, monkeypatch, kombu, redis_client+ ):+ # Fail closed: only the exact opt-in switches transport.+ monkeypatch.setenv("LOG_TRANSPORT", "rabbit")+ LogPublisher.publish("chan-1", _PAYLOAD)+ kombu.publish.assert_called_once()+ redis_client.rpush.assert_not_called()+++class TestFlagOn:+ @pytest.fixture(autouse=True)+ def _enable(self, monkeypatch):+ monkeypatch.setenv("LOG_TRANSPORT", "redis")+ monkeypatch.setenv("LOG_STREAM_QUEUE_NAME", "log_stream_queue")++ def test_pushes_an_envelope_and_never_touches_amqp(self, kombu, redis_client):+ assert LogPublisher.publish("chan-1", _PAYLOAD) is True+ kombu.publish.assert_not_called()++ queue, raw = redis_client.rpush.call_args[0]+ assert queue == "log_stream_queue"+ envelope = json.loads(raw)+ # The task name is what the consumer dispatches on. Without it a rename would+ # silently drop every log with nothing at the publish site to trace it from.+ assert envelope["task"] == LogProcessingTask.TASK_NAME+ assert envelope["kwargs"]["message"] == _PAYLOAD+ assert envelope["kwargs"]["user_session_id"] == "chan-1"+ assert envelope["kwargs"]["event"] == "logs:chan-1"++ def test_drops_at_capacity_rather_than_growing_until_redis_ooms(self, redis_client):+ redis_client.llen.return_value = 10_000+ assert LogPublisher.publish("chan-1", _PAYLOAD) is True+ redis_client.rpush.assert_not_called()++ def test_a_redis_failure_is_swallowed_not_raised(self, redis_client):+ # A logging fault must never surface into the execution that emitted it.+ redis_client.rpush.side_effect = RuntimeError("redis down")+ assert LogPublisher.publish("chan-1", _PAYLOAD) is False++ def test_unified_notification_still_stored(self, redis_client):+ with patch.object(LogPublisher, "store_for_unified_notification") as store:+ LogPublisher.publish("chan-1", _PAYLOAD)+ store.assert_called_once()+ assert store.call_args[0][0] == "logs:chan-1"+++class TestEnvelopeParity:+ """Both transports must carry the same kwargs, or the consumer behaves differently+ depending on how the log arrived."""++ def test_kwargs_match_across_transports(self, monkeypatch, kombu, redis_client):+ monkeypatch.delenv("LOG_TRANSPORT", raising=False)+ LogPublisher.publish("chan-1", _PAYLOAD)+ celery_kwargs = kombu.publish.call_args.kwargs["body"]["kwargs"]++ monkeypatch.setenv("LOG_TRANSPORT", "redis")+ LogPublisher.publish("chan-1", _PAYLOAD)+ redis_kwargs = json.loads(redis_client.rpush.call_args[0][1])["kwargs"]++ assert celery_kwargs == redis_kwargsdiff --git a/workers/tests/test_pg_finalization_fixes.py b/workers/tests/test_pg_finalization_fixes.pyindex 14981a1f5..064524459 100644--- a/workers/tests/test_pg_finalization_fixes.py+++ b/workers/tests/test_pg_finalization_fixes.py@@ -117,6 +117,57 @@ class TestRecoverStuckClientResponseShape:_RECOVERY_ENV = "WORKER_PG_STUCK_EXECUTION_RECOVERY_ENABLED"+_RECOVERY_SECONDS_ENV = "WORKER_PG_STUCK_EXECUTION_RECOVERY_SECONDS"+++class TestStuckRecoveryWindowDefault:+ """The window must be a DEFAULT, not something an operator sets.++ It used to inherit the barrier stuck-timeout (~2.5 h), which answers a different+ question: that one bounds how long a batch may make no progress, where a single+ file legitimately can take hours. This one only needs to outlast a callback that+ is about to fire — seconds.++ The threshold is not what makes recovery safe; the endpoint skips unless EVERY+ file is terminal, so a running execution is never a candidate however short this+ is. Inheriting hours therefore bought nothing and left a stranded execution dead+ for ~2.5 h — the common shape being a Celery run whose callback worker was removed+ by a deploy (300 s grace, against file-processing's 7200 s).++ Pinned as a default because production and on-prem have **no operator and no+ Flipt**: a value that only exists as an env override is one those environments+ never receive, and they are precisely the ones that cannot diagnose a hung+ execution themselves.+ """++ def _seconds(self, monkeypatch, value=None):+ from queue_backend.pg_queue import reaper as R++ monkeypatch.delenv(_RECOVERY_SECONDS_ENV, raising=False)+ if value is not None:+ monkeypatch.setenv(_RECOVERY_SECONDS_ENV, value)+ return R._positive_duration_from_env(+ _RECOVERY_SECONDS_ENV, R._DEFAULT_STUCK_RECOVERY_SECONDS, int+ )++ def test_default_is_ten_minutes(self, monkeypatch):+ assert self._seconds(monkeypatch) == 600++ def test_default_is_NOT_the_barrier_stuck_timeout(self, monkeypatch):+ """The regression this replaces. Any multi-hour value here means a stranded+ execution stays dead for hours in an environment with nobody to notice."""+ assert self._seconds(monkeypatch) < 3600++ def test_env_still_overrides_for_tuning(self, monkeypatch):+ assert self._seconds(monkeypatch, "120") == 120++ def test_a_nonsense_value_raises_rather_than_silently_defaulting(self, monkeypatch):+ """Loud-on-misconfig, matching the other duration knobs: silently falling back+ would hide a typo'd ConfigMap behind behaviour that looks deliberate."""+ import pytest++ with pytest.raises(ValueError):+ self._seconds(monkeypatch, "not-a-number")class TestStuckRecoveryDefaultOn:diff --git a/workers/tests/test_pg_metrics.py b/workers/tests/test_pg_metrics.pyindex 55f4d4775..e21bc2b67 100644--- a/workers/tests/test_pg_metrics.py+++ b/workers/tests/test_pg_metrics.py@@ -401,6 +401,8 @@ class TestReaperTickWiring:with (patch.object(reaper_mod, "recover_expired_barriers", return_value=[]),patch.object(reaper_mod, "rearm_expired_claims", return_value=0),+ patch.object(reaper_mod, "promote_due_scheduled", return_value=0),+ patch.object(reaper_mod, "dispatch_due_periodic_tasks", return_value=0),patch.object(reaper_mod, "refresh_queue_gauges") as refresh,):reaper.tick()@@ -414,6 +416,8 @@ class TestReaperTickWiring:with (patch.object(reaper_mod, "recover_expired_barriers", return_value=[]),patch.object(reaper_mod, "rearm_expired_claims", return_value=0),+ patch.object(reaper_mod, "promote_due_scheduled", return_value=0),+ patch.object(reaper_mod, "dispatch_due_periodic_tasks", return_value=0),patch.object(reaper_mod, "refresh_queue_gauges") as refresh,):reaper.tick()@@ -432,6 +436,8 @@ class TestReaperTickWiring:with (patch.object(reaper_mod, "recover_expired_barriers", return_value=[]),patch.object(reaper_mod, "rearm_expired_claims", return_value=0),+ patch.object(reaper_mod, "promote_due_scheduled", return_value=0),+ patch.object(reaper_mod, "dispatch_due_periodic_tasks", return_value=0),patch.object(reaper_mod, "refresh_queue_gauges", side_effect=RuntimeError("db")) as refresh,@@ -458,6 +464,8 @@ class TestReaperTickWiring:with (patch.object(reaper_mod, "recover_expired_barriers", return_value=[]),patch.object(reaper_mod, "rearm_expired_claims", return_value=0),+ patch.object(reaper_mod, "promote_due_scheduled", return_value=0),+ patch.object(reaper_mod, "dispatch_due_periodic_tasks", return_value=0),patch.object(reaper_mod, "refresh_queue_gauges") as refresh,):reaper.tick() # becomes leader, refresh #1@@ -479,6 +487,8 @@ class TestReaperTickWiring:with (patch.object(reaper_mod, "recover_expired_barriers", return_value=[]),patch.object(reaper_mod, "rearm_expired_claims", return_value=0),+ patch.object(reaper_mod, "promote_due_scheduled", return_value=0),+ patch.object(reaper_mod, "dispatch_due_periodic_tasks", return_value=0),patch.object(reaper_mod, "refresh_queue_gauges"),):reaper.tick() # becomes leader@@ -499,6 +509,8 @@ class TestReaperTickWiring:with (patch.object(reaper_mod, "recover_expired_barriers", return_value=[]),patch.object(reaper_mod, "rearm_expired_claims", return_value=0),+ patch.object(reaper_mod, "promote_due_scheduled", return_value=0),+ patch.object(reaper_mod, "dispatch_due_periodic_tasks", return_value=0),patch.object(reaper_mod, "refresh_queue_gauges"),):reaper.tick() # first leader tick sweeps immediately@@ -513,6 +525,8 @@ class TestReaperTickWiring:with (patch.object(reaper_mod, "recover_expired_barriers", return_value=[]),patch.object(reaper_mod, "rearm_expired_claims", return_value=0),+ patch.object(reaper_mod, "promote_due_scheduled", return_value=0),+ patch.object(reaper_mod, "dispatch_due_periodic_tasks", return_value=0),patch.object(reaper_mod, "refresh_queue_gauges"),):reaper.tick()diff --git a/workers/tests/test_pg_queue_client.py b/workers/tests/test_pg_queue_client.pyindex a79ae8208..3fa1a4baa 100644--- a/workers/tests/test_pg_queue_client.py+++ b/workers/tests/test_pg_queue_client.py@@ -26,8 +26,9 @@ from queue_backend.fairness import DEFAULT_PRIORITY, MAX_PRIORITY, MIN_PRIORITYfrom queue_backend.pg_queue import PgQueueClient, QueueMessagefrom queue_backend.pg_queue.client import _SEND_RETRY_BACKOFF_SECONDSfrom queue_backend.pg_queue.connection import create_pg_connection-from queue_backend.pg_queue.reaper import rearm_expired_claims+from queue_backend.pg_queue.reaper import promote_due_scheduled, rearm_expired_claimsfrom queue_backend.pg_queue.schema import qualified+from unstract.core.data_models import QueueMessageState# --- Unit: SQL shape against a mocked connection ---@@ -670,6 +671,82 @@ class TestPgQueueClientIntegration:assert rearm_expired_claims(pg_conn) == 0 # vt not expired → untouchedassert client.read(queue_name, vt_seconds=30, qty=10) == [] # still claimed+ def _schedule(self, pg_conn, queue_name, seconds_from_now):+ """Insert a deferred row the way the backend producer writes one."""+ state = QueueMessageState.SCHEDULED.value+ with pg_conn.cursor() as cur:+ cur.execute(+ f"INSERT INTO {qualified('pg_queue_message')} "+ "(queue_name, message, org_id, priority, enqueued_at, vt, read_ct, "+ " state, available_at) "+ "VALUES (%s, %s::jsonb, '', 5, now(), now(), 0, %s, "+ " now() + make_interval(secs => %s)) "+ "RETURNING msg_id",+ (queue_name, '{"n": 1}', state, seconds_from_now),+ )+ msg_id = cur.fetchone()[0]+ pg_conn.commit()+ return msg_id++ def test_scheduled_row_is_invisible_to_the_claim(self, pg_conn, queue_name):+ # The core delayed-visibility guarantee (UN-3843): a not-yet-due row must+ # never be handed to a consumer. It is enforced structurally — 'scheduled'+ # is absent from the claim's partial index — not by a time predicate.+ client = PgQueueClient(conn=pg_conn)+ self._schedule(pg_conn, queue_name, 3600)+ assert client.read(queue_name, vt_seconds=30, qty=10) == []++ def test_promotion_makes_a_due_row_claimable(self, pg_conn, queue_name):+ # ...and the reaper sweep is what releases it. Negative offset = already due.+ client = PgQueueClient(conn=pg_conn)+ msg_id = self._schedule(pg_conn, queue_name, -1)+ assert client.read(queue_name, vt_seconds=30, qty=10) == [] # still deferred+ assert promote_due_scheduled(pg_conn) == 1+ claimed = client.read(queue_name, vt_seconds=30, qty=10)+ assert [m.msg_id for m in claimed] == [msg_id]++ def test_promotion_leaves_not_yet_due_rows_alone(self, pg_conn, queue_name):+ # The sweep must not release early — "not before available_at" is the whole+ # contract. A bug here fires a staggered send all at once.+ self._schedule(pg_conn, queue_name, 3600)+ assert promote_due_scheduled(pg_conn) == 0+ assert PgQueueClient(conn=pg_conn).read(queue_name, vt_seconds=30, qty=10) == []++ def test_promotion_is_idempotent(self, pg_conn, queue_name):+ # A promoted row leaves the predicate, so a second tick is a no-op rather+ # than re-arming an already-claimed message back to 'ready' (double-run).+ client = PgQueueClient(conn=pg_conn)+ self._schedule(pg_conn, queue_name, -1)+ assert promote_due_scheduled(pg_conn) == 1+ assert promote_due_scheduled(pg_conn) == 0+ client.read(queue_name, vt_seconds=30, qty=10) # now claimed+ assert promote_due_scheduled(pg_conn) == 0 # claimed rows are not re-promoted++ def test_rearm_does_not_touch_scheduled_rows(self, pg_conn, queue_name):+ # The two sweeps share a table and both write state='ready'; re-arm keys on+ # 'claimed' only, so a deferred row can't be released by the crash path.+ self._schedule(pg_conn, queue_name, 3600)+ assert rearm_expired_claims(pg_conn) == 0+ assert PgQueueClient(conn=pg_conn).read(queue_name, vt_seconds=30, qty=10) == []++ def test_default_available_at_keeps_the_raw_insert_working(self, pg_conn, queue_name):+ # Deploy-safety guard: the workers' enqueue SQL does not mention+ # available_at, so the column MUST keep a DB-level default (migration 0002+ # re-adds the one Django's AddField drops). Without it every worker enqueue+ # fails with a not-null violation, in any deploy order.+ client = PgQueueClient(conn=pg_conn)+ msg_id = client.send(queue_name, {"n": 1})+ with pg_conn.cursor() as cur:+ cur.execute(+ f"SELECT available_at, state FROM {qualified('pg_queue_message')} "+ "WHERE msg_id = %s",+ (msg_id,),+ )+ available_at, state = cur.fetchone()+ assert state == QueueMessageState.READY.value+ assert available_at is not None+ assert client.read(queue_name, vt_seconds=30, qty=10) # immediately claimable+def test_state_check_constraint_matches_enum(self, pg_conn, queue_name):# Drift guard (mirrors test_db_check_constraint_matches_fairness_bounds):# the DB CheckConstraint must accept exactly the QueueMessageState valuesdiff --git a/workers/tests/test_pg_reaper.py b/workers/tests/test_pg_reaper.pyindex 4a6f56f55..add6d8c5c 100644--- a/workers/tests/test_pg_reaper.py+++ b/workers/tests/test_pg_reaper.py@@ -31,6 +31,7 @@ from queue_backend.pg_queue.reaper import (dedup_retention_from_env,reaper_interval_from_env,reaper_sweep_interval_from_env,+ promote_due_scheduled,rearm_expired_claims,recover_expired_barriers,sweep_expired_results,@@ -49,6 +50,7 @@ from queue_backend.pg_queue.schema import qualifieddef stub_scheduler_tick(monkeypatch):mock = MagicMock(return_value=0)monkeypatch.setattr(reaper_mod, "dispatch_due_schedules", mock)+ monkeypatch.setattr(reaper_mod, "dispatch_due_periodic_tasks", MagicMock(return_value=0))return mock@@ -63,6 +65,16 @@ def stub_queue_rearm(monkeypatch):return mock+# The leader tick also promotes due scheduled queue messages (UN-3843 delayed+# visibility). Same reason as the re-arm stub above: the leadership / connection+# tests drive dummy connections that would blow up on a real UPDATE.[email protected](autouse=True)+def stub_queue_promote(monkeypatch):+ mock = MagicMock(return_value=0)+ monkeypatch.setattr(reaper_mod, "promote_due_scheduled", mock)+ return mock++# The leader tick also runs the retention sweep (UN-3610). Stub the two sweep# helpers by default so the leadership / connection tests don't hit a real DELETE# on their dummy connections; the SQL-contract tests import the real helpers@@ -417,6 +429,82 @@ class TestQueueRearmTick:assert reaper.metrics.queue_rearm_failures._value.get() == 1+class TestQueuePromoteTick:+ """UN-3843: the leader promotes due scheduled queue messages each cycle, next to+ the re-arm sweep. Promotion SQL is in TestRetentionSweepSql; here we assert the+ wiring, gating, metric guard and error posture (the autouse+ ``stub_queue_promote`` patches the helper).++ Cadence matters more here than for the other sweeps: this is the DELIVERY path+ for every countdown/eta dispatch, so running it anywhere but the per-tick+ recovery block would add latency to each delayed message.+ """++ def _reaper(self, lease):+ return PgReaper(+ lease, interval_seconds=0.01, sweep_conn=object(), api_client=object()+ )++ def test_leader_runs_promotion(self, stub_queue_promote):+ reaper = self._reaper(_FakeLease(acquires=True, renews=True))+ with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]):+ reaper.tick()+ stub_queue_promote.assert_called_once()++ def test_standby_does_not_promote(self, stub_queue_promote):+ # Two promoters would race on the same rows; leadership is what serialises it.+ reaper = self._reaper(_FakeLease(acquires=False))+ with patch.object(reaper_mod, "recover_expired_barriers"):+ reaper.tick()+ stub_queue_promote.assert_not_called()++ def test_promotion_runs_after_rearm_before_schedule(+ self, stub_queue_rearm, stub_queue_promote, stub_scheduler_tick+ ):+ order = []+ reaper = self._reaper(_FakeLease(acquires=True, renews=True))+ stub_queue_rearm.side_effect = lambda *_: order.append("rearm") or 0+ stub_queue_promote.side_effect = lambda *_: order.append("promote") or 0+ stub_scheduler_tick.side_effect = lambda *_: order.append("schedule")+ with patch.object(+ reaper_mod,+ "recover_expired_barriers",+ side_effect=lambda *_, **__: order.append("recover") or [],+ ):+ reaper.tick()+ assert order == ["recover", "rearm", "promote", "schedule"]++ def test_metric_incremented_only_when_nonzero(self, stub_queue_promote):+ reaper = self._reaper(_FakeLease(acquires=True, renews=True))+ counter = reaper.metrics.queue_promoted+ with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]):+ stub_queue_promote.return_value = 0 # nothing due+ reaper.tick()+ assert counter._value.get() == 0 # guarded by `if promoted:`+ stub_queue_promote.return_value = 2+ reaper.tick()+ assert counter._value.get() == 2++ def test_promote_error_discards_conn_and_counts_failure(self, stub_queue_promote):+ # Failure must not be swallowed: the symptom of a silently-stalled promotion+ # sweep is delayed messages that simply never fire, with no error at the+ # enqueue site to trace it back from. Dedicated counter, discard, re-raise.+ reaper = PgReaper(+ _FakeLease(acquires=True, renews=True),+ interval_seconds=0.01,+ api_client=object(),+ )+ owned = MagicMock()+ owned.closed = False+ reaper._sweep_conn = owned+ stub_queue_promote.side_effect = psycopg2.OperationalError("db gone")+ with patch.object(reaper_mod, "recover_expired_barriers", return_value=[]):+ with pytest.raises(psycopg2.OperationalError):+ reaper.tick()+ assert reaper._sweep_conn is None # discarded+ assert reaper.metrics.queue_promote_failures._value.get() == 1++class TestRetentionSweepSql:"""The sweep helpers' SQL contract (mock cursor, no DB). These call the realhelpers (imported at module load), unaffected by the autouse stub which patches@@ -462,17 +550,31 @@ class TestRetentionSweepSql:assert "WHERE state = 'claimed' AND vt <= now()" in sqlconn.commit.assert_called_once()+ def test_promote_due_scheduled_sql(self):+ # UN-3843 delayed-visibility helper. The predicate is the whole contract:+ # 'scheduled' scopes it to deferred rows (so it can never re-arm a claimed+ # message), and `available_at <= now()` is what makes delivery "not before"+ # the requested time. Its integration test is Postgres-gated.+ conn, cur = self._conn_cur(2)+ assert promote_due_scheduled(conn) == 2+ sql = cur.execute.call_args[0][0]+ assert f"UPDATE {qualified('pg_queue_message')}" in sql+ assert "SET state = 'ready'" in sql+ assert "WHERE state = 'scheduled' AND available_at <= now()" in sql+ conn.commit.assert_called_once()+@pytest.mark.parametrize("sweep",[lambda conn: sweep_expired_results(conn),lambda conn: sweep_orphan_dedup(conn, 60),lambda conn: rearm_expired_claims(conn),+ lambda conn: promote_due_scheduled(conn),],- ids=["expired_results", "orphan_dedup", "rearm_claims"],+ ids=["expired_results", "orphan_dedup", "rearm_claims", "promote_scheduled"],)def test_sweep_rolls_back_on_error(self, sweep):- # Both helpers have their own try/except/rollback — exercise each.+ # Each helper owns its try/except/rollback — exercise every one.cur = MagicMock()cur.execute.side_effect = psycopg2.OperationalError("dead")conn = MagicMock()diff --git a/workers/tests/test_pg_scheduler.py b/workers/tests/test_pg_scheduler.pyindex 5086d3c12..e5ea53bc0 100644--- a/workers/tests/test_pg_scheduler.py+++ b/workers/tests/test_pg_scheduler.py@@ -7,6 +7,7 @@ tests seed + clean up their own rows under a unique org marker."""import datetime+import jsonimport uuidimport psycopg2@@ -16,11 +17,47 @@ from queue_backend.pg_queue.pg_scheduler import (SCHEDULER_QUEUE_NAME,_build_trigger_payload,compute_next_run,+ dispatch_due_periodic_tasks,dispatch_due_schedules,)class TestPureHelpers:+ def test_the_field_names_ARE_the_select_column_lists(self):+ """The one guarantee the _fields-derived SELECT cannot give itself.++ Deriving the column list from the type makes a REORDER harmless — there is no+ second list to drift from. It makes a RENAME silent in a new way: the query+ simply starts asking for a different column. Every test that would execute+ these queries needs live Postgres and SKIPS without it, so a rename would+ otherwise reach production unchallenged.++ Pinning the tuples turns that into a red test in the default lane, which forces+ whoever renames a field to confirm a matching migration in+ backend/pg_queue/models.py. Do NOT update these literals to match a rename+ without doing so — that is the mistake this exists to stop.+ """+ from queue_backend.pg_queue.pg_scheduler import _DuePeriodicTask, _DueSchedule++ assert _DueSchedule._fields == (+ "pipeline_id",+ "organization_id",+ "workflow_id",+ "pipeline_name",+ "cron_string",+ "next_run_at",+ )+ assert _DuePeriodicTask._fields == (+ "name",+ "task_name",+ "queue",+ "task_args",+ "task_kwargs",+ "org_id",+ "cron_string",+ "next_run_at",+ )+def test_next_run_is_strictly_after_base(self):base = datetime.datetime(2026, 6, 18, 10, 0, 0)# 09:00 daily — already past at 10:00, so the next is tomorrow 09:00.@@ -80,8 +117,16 @@ def _seed(conn, *, pg_owned, enabled, next_run_at, cron="0 9 * * *"):created_at, updated_at)VALUES (%s, %s, %s, %s, %s, %s, %s, NULL, %s, now(), now())""",- (pid, _MARKER, str(uuid.uuid4()), "Test ETL", cron, enabled, pg_owned,- next_run_at),+ (+ pid,+ _MARKER,+ str(uuid.uuid4()),+ "Test ETL",+ cron,+ enabled,+ pg_owned,+ next_run_at,+ ),)conn.commit()return pid@@ -111,7 +156,9 @@ def clean(pg_conn):"""Remove any rows this test created (the tick commits, so teardown must)."""yield pg_connwith pg_conn.cursor() as cur:- cur.execute("DELETE FROM pg_periodic_schedule WHERE organization_id = %s", (_MARKER,))+ cur.execute(+ "DELETE FROM pg_periodic_schedule WHERE organization_id = %s", (_MARKER,)+ )cur.execute("DELETE FROM pg_queue_message WHERE org_id = %s", (_MARKER,))pg_conn.commit()@@ -275,3 +322,188 @@ class _FailingCursor:if self._fail_when(sql):raise psycopg2.OperationalError("forced failure")return self._cur.execute(sql, params)+++# --- generic (non-pipeline) periodics: UN-3796 ---++_PT_MARKER = f"test_pgperiodic_{uuid.uuid4().hex[:8]}"+_PT_QUEUE = f"{_PT_MARKER}_queue"+++def _seed_periodic(+ conn,+ *,+ pg_owned,+ enabled,+ next_run_at,+ cron="0 9 * * *",+ queue=_PT_QUEUE,+ task_args=None,+ task_kwargs=None,+ org_id="",+):+ """Insert one pg_periodic_task row; returns its name."""+ name = f"{_PT_MARKER}_{uuid.uuid4().hex[:6]}"+ with conn.cursor() as cur:+ cur.execute(+ """+ INSERT INTO pg_periodic_task+ (name, task_name, queue, task_args, task_kwargs, org_id,+ cron_string, enabled, pg_owned, last_run_at, next_run_at,+ created_at, updated_at)+ VALUES (%s, %s, %s, %s::jsonb, %s::jsonb, %s, %s, %s, %s, NULL, %s,+ now(), now())+ """,+ (+ name,+ "dashboard_metrics.aggregate_from_sources",+ queue,+ json.dumps(task_args if task_args is not None else []),+ json.dumps(task_kwargs if task_kwargs is not None else {}),+ org_id,+ cron,+ enabled,+ pg_owned,+ next_run_at,+ ),+ )+ conn.commit()+ return name+++def _periodic_row(conn, name):+ with conn.cursor() as cur:+ cur.execute(+ "SELECT last_run_at, next_run_at, enabled FROM pg_periodic_task "+ "WHERE name = %s",+ (name,),+ )+ return cur.fetchone()+++def _messages_on(conn, queue):+ with conn.cursor() as cur:+ cur.execute(+ "SELECT message FROM pg_queue_message WHERE queue_name = %s", (queue,)+ )+ return [r[0] for r in cur.fetchall()]+++def clean_periodic(pg_conn):+ yield pg_conn+ with pg_conn.cursor() as cur:+ cur.execute(+ "DELETE FROM pg_periodic_task WHERE name LIKE %s", (f"{_PT_MARKER}%",)+ )+ cur.execute(+ "DELETE FROM pg_queue_message WHERE queue_name LIKE %s", (f"{_PT_MARKER}%",)+ )+ pg_conn.commit()+++class TestDispatchDuePeriodicTasks:+ """UN-3796 — the Beat replacement for everything that isn't a pipeline."""++ def test_due_owned_row_fires_onto_its_own_queue_and_advances(self, clean_periodic):+ conn = clean_periodic+ past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc)+ name = _seed_periodic(+ conn,+ pg_owned=True,+ enabled=True,+ next_run_at=past,+ task_kwargs={"retention_days": 30},+ )++ assert dispatch_due_periodic_tasks(conn) == 1++ # Enqueued onto the row's OWN queue — the reason this can't reuse the+ # pipeline dispatcher, which targets one fixed queue.+ msgs = _messages_on(conn, _PT_QUEUE)+ assert len(msgs) == 1+ assert msgs[0]["task_name"] == "dashboard_metrics.aggregate_from_sources"+ assert msgs[0]["kwargs"] == {"retention_days": 30}+ assert msgs[0]["queue"] == _PT_QUEUE++ last_run, next_run, _ = _periodic_row(conn, name)+ assert last_run is not None and next_run > last_run++ def test_org_id_is_carried_onto_the_queue_message(self, clean_periodic):+ # org_id is empty for every periodic today (these are global jobs), but the+ # column exists so a later unification with pg_periodic_schedule — which is+ # org-scoped — stays a data migration. Pin that the dispatcher actually+ # threads it through rather than hardcoding "".+ conn = clean_periodic+ past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc)+ _seed_periodic(+ conn, pg_owned=True, enabled=True, next_run_at=past, org_id="org_acme"+ )+ assert dispatch_due_periodic_tasks(conn) == 1+ with conn.cursor() as cur:+ cur.execute(+ "SELECT org_id FROM pg_queue_message WHERE queue_name = %s",+ (_PT_QUEUE,),+ )+ assert cur.fetchone()[0] == "org_acme"++ def test_first_observation_baselines_without_firing(self, clean_periodic):+ # No burst when a row is handed over: Beat fires a new schedule at its next+ # cron match, not immediately, and the hand-over must match that.+ conn = clean_periodic+ name = _seed_periodic(conn, pg_owned=True, enabled=True, next_run_at=None)++ assert dispatch_due_periodic_tasks(conn) == 0+ assert _messages_on(conn, _PT_QUEUE) == []+ last_run, next_run, _ = _periodic_row(conn, name)+ assert last_run is None and next_run is not None++ @pytest.mark.parametrize(+ "pg_owned,enabled", [(False, True), (True, False), (False, False)]+ )+ def test_unowned_or_disabled_rows_never_fire(self, clean_periodic, pg_owned, enabled):+ # Dark by default: until a row is explicitly adopted, Beat is the only firer.+ conn = clean_periodic+ past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc)+ _seed_periodic(conn, pg_owned=pg_owned, enabled=enabled, next_run_at=past)+ assert dispatch_due_periodic_tasks(conn) == 0+ assert _messages_on(conn, _PT_QUEUE) == []++ def test_not_yet_due_row_does_not_fire(self, clean_periodic):+ conn = clean_periodic+ future = datetime.datetime(2999, 1, 1, tzinfo=datetime.timezone.utc)+ _seed_periodic(conn, pg_owned=True, enabled=True, next_run_at=future)+ assert dispatch_due_periodic_tasks(conn) == 0+ assert _messages_on(conn, _PT_QUEUE) == []++ def test_invalid_cron_disables_the_row_instead_of_looping(self, clean_periodic):+ # Otherwise it is re-selected and re-logs a traceback every tick, forever.+ conn = clean_periodic+ past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc)+ name = _seed_periodic(+ conn, pg_owned=True, enabled=True, next_run_at=past, cron="not a cron"+ )+ assert dispatch_due_periodic_tasks(conn) == 0+ assert _periodic_row(conn, name)[2] is False # enabled -> False+ assert _messages_on(conn, _PT_QUEUE) == []++ def test_second_tick_does_not_refire_an_advanced_row(self, clean_periodic):+ # The enqueue and the next_run_at advance share one transaction, so a row+ # that fired is not due again until its next cron match.+ conn = clean_periodic+ past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc)+ _seed_periodic(conn, pg_owned=True, enabled=True, next_run_at=past)+ assert dispatch_due_periodic_tasks(conn) == 1+ assert dispatch_due_periodic_tasks(conn) == 0+ assert len(_messages_on(conn, _PT_QUEUE)) == 1++ def test_rows_are_isolated_from_each_other(self, clean_periodic):+ # A bad cron on one row must not stop the others firing that tick.+ conn = clean_periodic+ past = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc)+ _seed_periodic(+ conn, pg_owned=True, enabled=True, next_run_at=past, cron="garbage"+ )+ _seed_periodic(conn, pg_owned=True, enabled=True, next_run_at=past)+ assert dispatch_due_periodic_tasks(conn) == 1+ assert len(_messages_on(conn, _PT_QUEUE)) == 1diff --git a/workers/tests/test_pg_schema_drift.py b/workers/tests/test_pg_schema_drift.pyindex 2dbcbe2a3..384ae357a 100644--- a/workers/tests/test_pg_schema_drift.py+++ b/workers/tests/test_pg_schema_drift.py@@ -45,6 +45,11 @@ WORKER_SCHEMA_CONTRACT = {"read_ct","priority","state",+ # Delayed visibility (countdown/eta), migration 0002. Absent from this set+ # until now, while workers depend on it in raw SQL — so renaming or dropping+ # it would have kept this guard green and broken delayed delivery at runtime,+ # which is exactly the drift this file exists to catch.+ "available_at",},"pg_task_result": {"task_id",@@ -79,6 +84,24 @@ WORKER_SCHEMA_CONTRACT = {"created_at","updated_at",},+ # UN-3796 — generic (non-pipeline) Beat periodics. Sibling of the table above;+ # each row carries its own task/queue/args rather than the pipeline trigger's+ # one fixed shape.+ "pg_periodic_task": {+ "name",+ "task_name",+ "queue",+ "task_args",+ "task_kwargs",+ "org_id",+ "cron_string",+ "enabled",+ "pg_owned",+ "last_run_at",+ "next_run_at",+ "created_at",+ "updated_at",+ },}diff --git a/workers/tests/test_reaper_undispatched_sweep.py b/workers/tests/test_reaper_undispatched_sweep.pynew file mode 100644index 000000000..099c1eb82--- /dev/null+++ b/workers/tests/test_reaper_undispatched_sweep.py@@ -0,0 +1,94 @@+"""The reaper's trigger for the backend undispatched-execution sweep.++The sweep's *logic* lives in the backend (it touches WorkflowExecution, the rate+limiter and API storage — none of which a worker may reach). What the reaper+contributes is leader election, so exactly one instance runs it. These pin the+wiring, which is the part the backend tests cannot see:++ * it is CADENCE-GATED, not per-tick — the rows are already older than the grace+ period, so running it every 5s would be pure DB load;+ * a failure is SWALLOWED — unlike barrier recovery, this cleans up work that is+ already dead, and discarding the connection would defer schedule dispatch, which+ serves live traffic.+"""++from __future__ import annotations++from types import SimpleNamespace+from unittest.mock import MagicMock, patch++from queue_backend.pg_queue import reaper as reaper_mod+++def _reaper_with(api_client):+ """A PgReaper stand-in exposing only what _sweep_undispatched_executions uses.++ Built with __new__ so no DB/lease/metrics registry is required; the two collaborators+ the method actually touches are injected.+ """+ r = reaper_mod.PgReaper.__new__(reaper_mod.PgReaper)+ r._get_api_client = lambda: api_client # type: ignore[method-assign]+ r._metrics = MagicMock()+ return r+++class TestTheTriggerIsWiredToTheBackend:+ def test_it_calls_the_backend_sweep(self):+ api = MagicMock()+ # The REAL client returns the parsed body verbatim (typed `-> dict[str, Any]`),+ # not an object with `.data`. This fixture used to be SimpleNamespace(data=...),+ # a shape nothing in the call chain produces — so it asserted inc(3) while+ # production read `.data` off a dict and always incremented by 0. The test+ # passed against a fiction and hid a dead metric.+ api.sweep_undispatched_executions.return_value = {"swept": 3}+ r = _reaper_with(api)+ r._sweep_undispatched_executions()+ api.sweep_undispatched_executions.assert_called_once_with()+ # Counted by how many were recovered, not "one sweep ran" — the rate is the+ # signal that requests are dying before dispatch.+ r._metrics.undispatched_swept.inc.assert_called_once_with(3)++ def test_a_zero_result_is_not_an_error(self):+ """The steady state. Must stay quiet, not log every 5 minutes forever."""+ api = MagicMock()+ api.sweep_undispatched_executions.return_value = {"swept": 0}+ r = _reaper_with(api)+ r._sweep_undispatched_executions() # no raise+ # A zero must NOT touch the counter: inc(0) is harmless but a nonzero rate is+ # the alert signal, so keep the series clean.+ r._metrics.undispatched_swept.inc.assert_not_called()++ def test_a_missing_or_odd_payload_does_not_raise(self):+ """A backend on an older image returns no `swept` key — that must not take+ down a leader tick that also dispatches schedules.+ """+ for payload in (None, {}, {"unexpected": 1}, SimpleNamespace(data={"swept": 9})):+ api = MagicMock()+ api.sweep_undispatched_executions.return_value = payload+ _reaper_with(api)._sweep_undispatched_executions() # no raise+++class TestAFailureCannotBreakTheTick:+ def test_an_api_error_is_swallowed(self):+ """Barrier recovery re-raises and discards the connection because a stranded+ RUNNING execution is urgent. This is the opposite case — the executions are+ already dead — and raising here would abort the tick and defer schedule+ dispatch for live traffic. Cleanup must never outrank scheduling.+ """+ api = MagicMock()+ api.sweep_undispatched_executions.side_effect = RuntimeError("backend down")+ r = _reaper_with(api)+ r._sweep_undispatched_executions() # must not raise+ # Swallowed, but counted — otherwise a persistently failing sweep is invisible+ # and PENDING rows accumulate unrecovered.+ r._metrics.undispatched_sweep_failures.inc.assert_called_once()++ def test_the_failure_is_logged_so_it_is_not_silent(self):+ """Swallowed is not the same as hidden — a persistently failing sweep means+ PENDING rows accumulate, and that has to be visible.+ """+ api = MagicMock()+ api.sweep_undispatched_executions.side_effect = RuntimeError("backend down")+ with patch.object(reaper_mod.logger, "warning") as warn:+ _reaper_with(api)._sweep_undispatched_executions()+ assert warn.calleddiff --git a/workers/tests/test_worker_task_module_imports.py b/workers/tests/test_worker_task_module_imports.pynew file mode 100644index 000000000..df5494350--- /dev/null+++ b/workers/tests/test_worker_task_module_imports.py@@ -0,0 +1,156 @@+"""Every worker's tasks.py must import under BOTH mechanisms worker.py uses.++This exists because a bare ``import dashboard_metrics_tasks`` in scheduler/tasks.py+shipped and crash-looped **worker-general** in integration:++ File "/app/general/tasks.py", line 16, in <module>+ from scheduler.tasks import execute_pipeline_task_v2+ File "/app/scheduler/tasks.py", line 14, in <module>+ import dashboard_metrics_tasks+ ModuleNotFoundError: No module named 'dashboard_metrics_tasks'++The bare form resolves only when ``worker.py`` loads ``scheduler/tasks.py`` BY PATH with+``/app/scheduler`` appended to ``sys.path``. It does not resolve when another worker+imports it as a package module — ``general/tasks.py`` does exactly that, and then+``/app/scheduler`` is not on the path. Flag-off; PG is not involved.++The entire workers suite passed while this was broken, because nothing exercised the two+import mechanisms. That is the gap these close.++**Why subprocesses.** Importing the same file under two names (``dashboard_metrics_tasks``+and ``scheduler.dashboard_metrics_tasks``) creates two module objects and duplicate+Celery task registrations, which breaks other suites that patch one copy — doing this+in-process made test_dashboard_metrics_tasks fail. A fresh interpreter per mechanism is+both properly isolated and a truer reproduction of what a booting worker does.+"""++from __future__ import annotations++import json+import subprocess+import sys+from pathlib import Path++import pytest++_WORKERS_ROOT = Path(__file__).resolve().parent.parent++# Discovered, not hardcoded, so a new worker is covered automatically.+_TASK_MODULES = sorted(+ p.parent.name+ for p in _WORKERS_ROOT.glob("*/tasks.py")+ if not p.parent.name.startswith((".", "_")) and p.parent.name != "tests"+)++# Mechanism 2 — `from <worker>.tasks import ...`, the way general/tasks.py reaches+# scheduler/tasks.py. Only the workers root is on sys.path, never the worker's own+# directory: that is the situation that broke.+_PACKAGE_IMPORT_PROBE = """+import importlib, json, sys, traceback+sys.path.insert(0, {root!r})+failures = {{}}+for name in {workers!r}:+ try:+ importlib.import_module(name + ".tasks")+ except Exception:+ failures[name] = traceback.format_exc().strip().splitlines()[-1]+print("RESULT" + json.dumps(failures))+"""++# Mechanism 1 — worker.py's spec_from_file_location load with the worker dir appended,+# i.e. how a worker loads its own tasks.py at boot.+_PATH_IMPORT_PROBE = """+import importlib.util, json, sys, traceback+sys.path.insert(0, {root!r})+failures = {{}}+for name in {workers!r}:+ wd = {root!r} + "/" + name+ sys.path.append(wd)+ try:+ spec = importlib.util.spec_from_file_location("tasks_" + name.replace("-", "_"),+ wd + "/tasks.py")+ mod = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(mod)+ except Exception:+ failures[name] = traceback.format_exc().strip().splitlines()[-1]+ finally:+ sys.path.remove(wd)+print("RESULT" + json.dumps(failures))+"""+++def _run_probe(source: str, workers: list[str] | None = None) -> dict[str, str]:+ proc = subprocess.run(+ [sys.executable, "-c",+ source.format(root=str(_WORKERS_ROOT), workers=workers or _TASK_MODULES)],+ capture_output=True,+ text=True,+ cwd=str(_WORKERS_ROOT),+ timeout=300,+ )+ marker = [ln for ln in proc.stdout.splitlines() if ln.startswith("RESULT")]+ if not marker:+ pytest.fail(f"probe did not complete.\nstdout:\n{proc.stdout}\nstderr:\n{proc.stderr}")+ return json.loads(marker[-1][len("RESULT") :])+++def test_discovery_found_the_workers() -> None:+ """Guard against the glob matching nothing and every case passing vacuously."""+ assert len(_TASK_MODULES) >= 5, _TASK_MODULES+ assert "scheduler" in _TASK_MODULES+ assert "general" in _TASK_MODULES+++# A hyphenated directory (api-deployment) can never be imported package-style — it is+# not a valid Python identifier — so only the file-path mechanism applies to it.+_PACKAGE_IMPORTABLE = [w for w in _TASK_MODULES if w.isidentifier()]+++def test_every_tasks_module_imports_as_a_package() -> None:+ """The mechanism that broke: worker's own dir NOT on sys.path."""+ failures = _run_probe(_PACKAGE_IMPORT_PROBE, workers=_PACKAGE_IMPORTABLE)+ assert not failures, "package-style import failed:\n" + "\n".join(+ f" {k}: {v}" for k, v in sorted(failures.items())+ )+++def test_every_tasks_module_imports_by_file_path() -> None:+ """The mechanism worker.py uses for a worker's own tasks.py."""+ failures = _run_probe(_PATH_IMPORT_PROBE)+ assert not failures, "file-path import failed:\n" + "\n".join(+ f" {k}: {v}" for k, v in sorted(failures.items())+ )+++def test_scheduler_registers_the_metrics_proxies_under_their_wire_names() -> None:+ """The broken import is a SIDE-EFFECT import — it must actually register.++ Asserting only that scheduler.tasks imports would still pass if someone 'fixed' it+ by deleting the import, silently unregistering the three dashboard-metrics proxies+ that the PG metrics consumer resolves BY NAME.+ """+ source = """+import importlib, json, sys+sys.path.insert(0, {root!r})+m = importlib.import_module("scheduler.dashboard_metrics_tasks")+print("RESULT" + json.dumps({{+ "aggregate": m.dashboard_metrics_aggregate.name,+ "hourly": m.dashboard_metrics_cleanup_hourly.name,+ "daily": m.dashboard_metrics_cleanup_daily.name,+}}))+"""+ proc = subprocess.run(+ [sys.executable, "-c", source.format(root=str(_WORKERS_ROOT))],+ capture_output=True,+ text=True,+ cwd=str(_WORKERS_ROOT),+ timeout=300,+ )+ marker = [ln for ln in proc.stdout.splitlines() if ln.startswith("RESULT")]+ assert marker, f"probe failed:\n{proc.stdout}\n{proc.stderr}"+ names = json.loads(marker[-1][len("RESULT") :])+ assert names == {+ "aggregate": "dashboard_metrics.aggregate_from_sources",+ "hourly": "dashboard_metrics.cleanup_hourly_data",+ "daily": "dashboard_metrics.cleanup_daily_data",+ }