LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Tests for :class:`queue_backend.pg_barrier.PgBarrier`.
Three layers:
1. **Protocol / TTL** — no DB, no Celery.
2. **Enqueue + link/abort tasks** — a real autocommit Postgres connection is
injected into the module thread-local (the barrier's SQL runs for real); the
Celery header-task dispatch + callback are mocked. Skips if Postgres is
unreachable or the ``pg_barrier_state`` migration is unapplied.
3. **Atomicity** — two real connections race the decrement SQL directly.
"""
from __future__ import annotations
import os
from unittest.mock import MagicMock, patch
import psycopg2
import psycopg2.extensions
import pytest
from queue_backend import barrier as barrier_mod
from queue_backend import pg_barrier
from queue_backend.fairness import FAIRNESS_HEADER_NAME, FairnessKey, WorkloadType
from queue_backend.handle import BarrierHandle
from queue_backend.pg_barrier import (
PgBarrier,
_barrier_pg_decrement,
_fire_barrier_callback,
barrier_pg_abort,
claim_batch,
run_batch_with_barrier,
try_claim_orchestration,
)
from queue_backend.pg_queue.connection import create_pg_connection
from queue_backend.routing import QueueBackend
def _pg_header(task_name="process_file_batch", args=None, queue="file_processing"):
"""A Celery-Signature-shaped stub for the PG-fan-out path (.task/.args/.kwargs/.options)."""
sig = MagicMock(name="pg_header_signature")
sig.task = task_name
sig.args = args if args is not None else [{"file": "f1"}]
sig.kwargs = {}
sig.options = {"queue": queue}
return sig
_CALLBACK = {
"task_name": "process_batch_callback_api",
"kwargs": {"execution_id": "exec-1", "pipeline_id": "pipe-1"},
"queue": "general",
"fairness_headers": None,
}
def _mock_header_task():
"""A header-task Signature for the fan-out, PG-shaped.
Kept as a two-tuple so the many ``_mock_header_task()[0]`` call sites read
unchanged; the second element is now unused (it was the ``.clone()`` mock the
Celery ``.link`` path recorded against, and that path is gone).
"""
sig = _pg_header()
return sig, None
# --- Layer 1: protocol shape + TTL (no DB) ---
class TestPgBarrierProtocolShape:
def test_satisfies_barrier_protocol(self):
barrier: barrier_mod.Barrier = PgBarrier()
assert callable(getattr(barrier, "enqueue", None))
def test_handle_satisfies_barrier_handle(self):
handle: BarrierHandle = pg_barrier._PgBarrierHandle(id="exec-1")
assert handle.id == "exec-1"
assert isinstance(handle.id, str)
class TestTtlEnv:
# PgBarrier shares barrier.barrier_ttl_seconds() with the Redis backend.
def test_default_is_six_hours(self, monkeypatch):
monkeypatch.delenv("WORKER_BARRIER_KEY_TTL_SECONDS", raising=False)
assert barrier_mod.barrier_ttl_seconds() == 6 * 60 * 60
def test_overridable(self, monkeypatch):
monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "120")
assert barrier_mod.barrier_ttl_seconds() == 120
@pytest.mark.parametrize("bad", ["abc", "0", "-5"])
def test_invalid_raises(self, monkeypatch, bad):
monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", bad)
with pytest.raises(ValueError, match="WORKER_BARRIER_KEY_TTL_SECONDS"):
barrier_mod.barrier_ttl_seconds()
class TestStuckTimeoutEnv:
# The PG barrier's sliding last_progress_at window (UN-3661) — distinct from the
# Redis-shared TTL above. Default 2.5h, in Celery's FILE_PROCESSING band (2h–3h).
def test_default_is_two_and_half_hours(self, monkeypatch):
monkeypatch.delenv("WORKER_PG_BATCH_STUCK_TIMEOUT_SECONDS", raising=False)
assert barrier_mod.barrier_stuck_timeout_seconds() == 9000
def test_overridable(self, monkeypatch):
monkeypatch.setenv("WORKER_PG_BATCH_STUCK_TIMEOUT_SECONDS", "120")
assert barrier_mod.barrier_stuck_timeout_seconds() == 120
@pytest.mark.parametrize("bad", ["abc", "0", "-5"])
def test_invalid_raises(self, monkeypatch, bad):
monkeypatch.setenv("WORKER_PG_BATCH_STUCK_TIMEOUT_SECONDS", bad)
with pytest.raises(ValueError, match="WORKER_PG_BATCH_STUCK_TIMEOUT_SECONDS"):
barrier_mod.barrier_stuck_timeout_seconds()
class TestEnqueueShortCircuits:
def test_empty_header_returns_none(self):
# Returns before any DB / dispatch.
assert (
PgBarrier().enqueue(
[],
callback_task_name="cb",
callback_kwargs={"execution_id": "e"},
callback_queue="general",
app_instance=None,
)
is None
)
def test_missing_execution_id_raises(self):
barrier = PgBarrier()
header = _mock_header_task()[0]
with pytest.raises(ValueError, match="execution_id"):
barrier.enqueue(
[header],
callback_task_name="cb",
callback_kwargs={}, # no execution_id
callback_queue="general",
app_instance=None,
)
# --- Idempotent-write reconnect-retry (no DB) ---
_UNSET = object()
class _FakeCursorCtx:
def __init__(self, on_execute, fetchone_result=_UNSET):
self._on_execute = on_execute
self._fetchone_result = fetchone_result
def __enter__(self):
cur = MagicMock(name="cursor")
cur.execute.side_effect = self._on_execute
# Default (unset): fetchone() returns a truthy MagicMock (claim won). Pass
# fetchone_result=None to exercise the "lost" answer (claim already held).
if self._fetchone_result is not _UNSET:
cur.fetchone.return_value = self._fetchone_result
return cur
def __exit__(self, *exc):
return False
class _FakeConn:
"""Minimal psycopg2-connection stub. ``execute_error`` (if set) is raised by
every cursor.execute on this connection — simulating a stale/dead socket.
``commit_error`` (if set) is raised by ``commit()`` after the execute lands —
the ambiguous "reaped during commit" case the decrement must NOT retry.
"""
def __init__(self, *, execute_error=None, commit_error=None, fetchone_result=_UNSET):
self.closed = False
self._execute_error = execute_error
self._commit_error = commit_error
self._fetchone_result = fetchone_result
self.commits = 0
self.rollbacks = 0
self.executes = 0
def cursor(self):
def _on_execute(*_a, **_k):
self.executes += 1
if self._execute_error is not None:
raise self._execute_error
return _FakeCursorCtx(_on_execute, self._fetchone_result)
def commit(self):
self.commits += 1
if self._commit_error is not None:
raise self._commit_error
def rollback(self):
self.rollbacks += 1
def close(self):
self.closed = True
def get_transaction_status(self):
# The decrement entry-guard checks this; a stub conn is always idle (each
# _cursor use commits), so it never trips the open-transaction guard.
return psycopg2.extensions.TRANSACTION_STATUS_IDLE
@pytest.fixture
def _clean_local():
"""Ensure the module thread-local connection is reset around each test."""
pg_barrier._local.conn = None
yield
pg_barrier._local.conn = None
class TestIdempotentWriteRetry:
"""`_run_idempotent_pre_dispatch_write` self-heals a stale cached connection
with ONE retry — the fix for the `server closed the connection unexpectedly`
abort at barrier enqueue. It must retry ONLY on connection errors, stay
bounded, and never silently swallow a real (non-connection) failure.
"""
@pytest.fixture(autouse=True)
def _no_sleep(self, monkeypatch):
# The retry backoff is real (0.5s); skip the wall-clock wait in tests.
monkeypatch.setattr(pg_barrier.time, "sleep", lambda *_a, **_k: None)
@pytest.mark.parametrize(
"exc",
[
psycopg2.OperationalError("server closed the connection unexpectedly"),
psycopg2.InterfaceError("connection already closed"),
],
ids=["OperationalError", "InterfaceError"],
)
def test_retries_once_on_dead_connection_then_succeeds(
self, _clean_local, monkeypatch, caplog, exc
):
# InterfaceError is the more common stale-socket symptom, so both arms of
# the (OperationalError, InterfaceError) catch must self-heal.
dead = _FakeConn(execute_error=exc)
healthy = _FakeConn()
pg_barrier._local.conn = dead
# On reconnect (_cursor discarded the dead conn), hand back the healthy one.
monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: healthy)
attempts = []
def op(cur):
attempts.append(1)
cur.execute("UPSERT ...", ("x",))
with caplog.at_level("WARNING"):
pg_barrier._run_idempotent_pre_dispatch_write(op, what="test")
assert len(attempts) == 2 # first failed mid-execute, retry succeeded
assert dead.executes == 1 and healthy.executes == 1 # exactly one extra try
assert dead.commits == 0 # no partial commit on the failed attempt
assert dead.closed is True # stale conn discarded
assert healthy.commits == 1 # committed exactly once, on the retry
assert pg_barrier._local.conn is healthy
# logs on retry (not on success) and names the real error, not a guess.
assert "reconnecting and retrying" in caplog.text
assert type(exc).__name__ in caplog.text
def test_does_not_retry_non_connection_error(self, _clean_local, monkeypatch):
healthy = _FakeConn()
pg_barrier._local.conn = healthy
reconnects = []
monkeypatch.setattr(
pg_barrier,
"create_pg_connection",
lambda **_k: reconnects.append(1) or _FakeConn(),
)
def op(cur):
raise ValueError("not a connection problem")
with pytest.raises(ValueError, match="not a connection problem"):
pg_barrier._run_idempotent_pre_dispatch_write(op, what="test")
assert reconnects == [] # a real error must surface immediately, no retry
def test_reraises_after_exhausting_attempts(self, _clean_local, monkeypatch):
err = psycopg2.OperationalError("still down")
# Every connection (initial + reconnect) is dead → both attempts fail.
monkeypatch.setattr(
pg_barrier, "create_pg_connection", lambda **_k: _FakeConn(execute_error=err)
)
with pytest.raises(psycopg2.OperationalError, match="still down"):
pg_barrier._run_idempotent_pre_dispatch_write(
lambda cur: cur.execute("X"), what="test"
)
class TestDecrementPhaseSplitRetry:
"""`_apply_decrement` is the NON-idempotent barrier decrement, so unlike the
idempotent pre-dispatch write it CANNOT retry freely. It self-heals exactly
one provably-safe case — an execute-phase failure on a *cached* connection
(the PgBouncer idle-reap: the statement never reached the server, so nothing
committed) — and refuses every other: a commit-phase failure is ambiguous
(the server may have applied it) and a fresh-conn failure is a real DB error.
The count is therefore never double-applied. (UN-3660)
"""
@pytest.fixture
def sleeps(self, monkeypatch):
# Record (and skip) the retry backoff so a test can assert it fired exactly
# when — and only when — a retry happens. Deleting the time.sleep in
# _apply_decrement, or a refactor that starts hammering a struggling DB,
# changes this list.
calls: list[float] = []
monkeypatch.setattr(pg_barrier.time, "sleep", calls.append)
return calls
@pytest.mark.parametrize(
"exc",
[
psycopg2.OperationalError("server closed the connection unexpectedly"),
psycopg2.InterfaceError("connection already closed"),
],
ids=["OperationalError", "InterfaceError"],
)
def test_execute_phase_reaped_cached_conn_retries_once(
self, _clean_local, monkeypatch, caplog, sleeps, exc
):
# The idle-reap: a cached conn fails mid-execute (never committed), so the
# decrement reconnects and re-applies exactly once on a healthy conn.
dead = _FakeConn(execute_error=exc)
healthy = _FakeConn()
pg_barrier._local.conn = dead # cached → "reused"
monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: healthy)
with caplog.at_level("WARNING"):
pg_barrier._apply_decrement("exec-1", '{"ok": true}', reused=True)
assert dead.executes == 1 and healthy.executes == 1 # exactly one extra try
assert dead.commits == 0 # the reaped attempt never committed
assert dead.closed is True # stale conn discarded
assert healthy.commits == 1 # committed exactly once, on the retry
assert pg_barrier._local.conn is healthy
assert sleeps == [pg_barrier._BARRIER_RETRY_BACKOFF_SECONDS] # backoff fired once
assert "execute failed on a cached connection" in caplog.text
assert type(exc).__name__ in caplog.text
def test_commit_phase_failure_is_not_retried(self, _clean_local, monkeypatch, sleeps):
# A commit failure is AMBIGUOUS (the server may have applied it) → must
# NOT retry, or the decrement could land twice. It propagates; the dead
# conn is discarded; no reconnect is attempted.
commit_err = psycopg2.OperationalError("server closed during commit")
conn = _FakeConn(commit_error=commit_err)
pg_barrier._local.conn = conn
reconnects = []
monkeypatch.setattr(
pg_barrier,
"create_pg_connection",
lambda **_k: reconnects.append(1) or _FakeConn(),
)
with pytest.raises(psycopg2.OperationalError, match="during commit"):
pg_barrier._apply_decrement("exec-2", '{"ok": true}', reused=True)
assert conn.executes == 1 # the UPDATE ran exactly once
assert conn.commits == 1 # commit was attempted exactly once
assert reconnects == [] # NEVER reconnected/retried after a commit failure
assert conn.closed is True # the dead conn was discarded
assert sleeps == [] # no backoff — a commit failure is not retried
def test_fresh_conn_execute_failure_is_not_retried(
self, _clean_local, monkeypatch, sleeps
):
# reused=False (the production wrapper passes this when no conn was cached
# before the entry guard). A failure on a fresh conn is a genuine DB error,
# not an idle-reap, so the reused-guard skips the retry — even though it is
# a connection-dead error — and it surfaces immediately.
err = psycopg2.OperationalError("db down")
conn = _FakeConn(execute_error=err)
pg_barrier._local.conn = conn
reconnects = []
monkeypatch.setattr(
pg_barrier,
"create_pg_connection",
lambda **_k: reconnects.append(1) or _FakeConn(),
)
with pytest.raises(psycopg2.OperationalError, match="db down"):
pg_barrier._apply_decrement("exec-3", '{"ok": true}', reused=False)
assert reconnects == [] # fresh-conn death is not retried
assert sleeps == [] # …so no backoff either
def test_non_connection_error_is_not_retried(self, _clean_local, monkeypatch, sleeps):
# A DataError (e.g. a NUL byte rejected by the jsonb cast) is not a
# connection death → propagate immediately so the caller tears the barrier
# down. A live conn after a logical error is NOT discarded.
conn = _FakeConn(execute_error=psycopg2.DataError("invalid byte 0x00"))
pg_barrier._local.conn = conn
reconnects = []
monkeypatch.setattr(
pg_barrier,
"create_pg_connection",
lambda **_k: reconnects.append(1) or _FakeConn(),
)
with pytest.raises(psycopg2.DataError):
pg_barrier._apply_decrement("exec-4", '{"ok": true}', reused=True)
assert reconnects == [] # no retry on a logical/data error
assert conn.closed is False # a live conn after a data error is kept
assert sleeps == [] # not retried → no backoff
def test_reraises_after_one_retry(self, _clean_local, monkeypatch, sleeps):
# The one-shot bound: a reused-conn idle-reap retries ONCE; if the
# reconnect target also dies on execute, re-raise rather than loop. On
# attempt 2 the attempt bound (attempt < _BARRIER_DECREMENT_ATTEMPTS) is
# what refuses the next retry — note reused is still True (the caller's
# entry-time value), so bumping the attempt constant would NOT add
# self-heals unless the reconnect logic also re-evaluated freshness.
err = psycopg2.OperationalError("still down")
pg_barrier._local.conn = _FakeConn(execute_error=err)
reconnects = []
monkeypatch.setattr(
pg_barrier,
"create_pg_connection",
lambda **_k: reconnects.append(1) or _FakeConn(execute_error=err),
)
with pytest.raises(psycopg2.OperationalError, match="still down"):
pg_barrier._apply_decrement("exec-5", '{"ok": true}', reused=True)
assert len(reconnects) == 1 # exactly one reconnect, then gave up
assert sleeps == [pg_barrier._BARRIER_RETRY_BACKOFF_SECONDS] # one backoff
def test_wrapper_fresh_conn_not_retried_end_to_end(
self, _clean_local, monkeypatch, sleeps
):
# Through the PRODUCTION entry (_barrier_pg_decrement), not _apply_decrement
# directly: with no cached conn, the entry-guard's _get_conn() creates a
# fresh one — but _barrier_pg_decrement samples `reused` BEFORE the guard,
# so it threads reused=False and a genuine fresh-conn DB error is NOT
# retried. Pins the sample-before-the-guard wiring the direct-call tests
# can't see (the guard would otherwise leave _local.conn always populated).
err = psycopg2.OperationalError("db down")
creates = []
monkeypatch.setattr(
pg_barrier,
"create_pg_connection",
lambda **_k: creates.append(c := _FakeConn(execute_error=err)) or c,
)
pg_barrier._local.conn = None # nothing cached → wrapper samples reused=False
with pytest.raises(psycopg2.OperationalError, match="db down"):
_barrier_pg_decrement(
{"f": 1}, execution_id="exec-FRESH", callback_descriptor=_CALLBACK
)
assert len(creates) == 1 # the guard created one; NO retry reconnect
assert sleeps == [] # a fresh-conn death is not retried → no backoff
def test_oversized_result_propagates_without_releasing_the_handle(
self, _clean_local, monkeypatch
):
# UN-4126, gating-lane cover for the barrier sink. Two properties:
#
# 1. `ProgramLimitExceeded` ("index row size exceeds maximum", SQLSTATE
# 54) subclasses OperationalError, so a `DataError`-only net would let
# it through without the specific, actionable log.
# 2. The decrement must NOT delete the barrier row itself. That row is
# the reaper's only recovery handle, and `run_batch_with_barrier`
# releases it only after confirming the execution is terminal. A
# teardown here pre-empts that ordering and, when the mark cannot be
# confirmed, strands the execution permanently — strictly worse than
# the hang-to-expiry it would be avoiding.
err = psycopg2.errors.ProgramLimitExceeded("index row size exceeds maximum")
monkeypatch.setattr(
pg_barrier,
"create_pg_connection",
lambda **_k: _FakeConn(execute_error=err),
)
deleted: list[str] = []
monkeypatch.setattr(
pg_barrier, "_delete_barrier", lambda eid: deleted.append(eid)
)
pg_barrier._local.conn = None
with pytest.raises(psycopg2.errors.ProgramLimitExceeded):
_barrier_pg_decrement(
{"f": "x" * 10}, execution_id="exec-BIG", callback_descriptor=_CALLBACK
)
assert deleted == [], "released the reaper's handle before the caller could mark"
def test_unencodable_result_propagates_without_releasing_the_handle(
self, _clean_local, monkeypatch
):
# The encode-side twin of the test above: allow_nan=False puts NaN on the
# encode branch, which must follow the same rule — log, propagate, and
# leave teardown to the caller's mark-then-release ordering.
monkeypatch.setattr(
pg_barrier, "create_pg_connection", lambda **_k: _FakeConn()
)
deleted: list[str] = []
monkeypatch.setattr(
pg_barrier, "_delete_barrier", lambda eid: deleted.append(eid)
)
pg_barrier._local.conn = None
with pytest.raises(ValueError):
_barrier_pg_decrement(
{"f": float("nan")},
execution_id="exec-NaN",
callback_descriptor=_CALLBACK,
)
assert deleted == [], "released the reaper's handle before the caller could mark"
@pytest.mark.integration
def test_create_pg_connection_is_non_autocommit():
"""The decrement phase-split's exactly-once safety rests on the connection
being non-autocommit (an uncommitted UPDATE rolls back on disconnect, so an
execute-phase failure is never durable). Pin that at its SOURCE — a future
``conn.autocommit = True`` in ``create_pg_connection`` would silently
reintroduce the double-count the split exists to prevent, and no other test
would catch it (the barrier_db fixture sets autocommit itself).
Opens a real connection inline (no fixture), so it's marked ``integration``
explicitly — the collection hook keys off fixture names and wouldn't catch
it, which would otherwise leave it in the DB-free unit lane.
"""
os.environ.setdefault("TEST_DB_HOST", "127.0.0.1")
try:
conn = create_pg_connection(env_prefix="TEST_DB_")
except psycopg2.OperationalError as exc:
pytest.skip(f"Postgres not reachable: {exc}")
try:
assert conn.autocommit is False
finally:
conn.close()
class TestClaimOrchestrationSchemaGuard:
"""``try_claim_orchestration``-specific: a schema behind the code fails fast with
an actionable message. The shared execute-phase reconnect-retry is covered once,
over both claim callers, by :class:`TestReturningClaimReconnect`.
"""
@pytest.mark.parametrize(
"exc_cls",
[psycopg2.errors.UndefinedTable, psycopg2.errors.UndefinedColumn],
ids=["table-missing (0012)", "column-missing (0013)"],
)
def test_schema_behind_raises_actionable_error(self, _clean_local, exc_cls):
# A schema behind the code — 0012 not run (no table) OR 0012-but-not-0013
# (table, no organization_id column) — must fail fast with an actionable
# message, NOT the generic per-execution stack trace, and NOT proceed.
pg_barrier._local.conn = _FakeConn(execute_error=exc_cls("schema behind"))
with pytest.raises(RuntimeError, match="schema is out of date"):
try_claim_orchestration("exec-1", "org-1")
@pytest.mark.parametrize(
"claim",
[
pytest.param(lambda: claim_batch("exec-1", 0), id="claim_batch"),
pytest.param(
lambda: try_claim_orchestration("exec-1", "org-1"),
id="try_claim_orchestration",
),
],
)
class TestReturningClaimReconnect:
"""The shared phase-split reconnect-retry
(``_run_returning_claim_with_reconnect``), exercised through BOTH RETURNING-claim
callers. Self-heals ONLY an execute-phase failure on a *cached* connection — the
idle reap that stranded scheduled ETL batches (UN-3684: a barrier connection idle
~30 min between pipeline runs, past ``server_idle_timeout``; the INSERT never
committed, so the retry's win/lost answer is authoritative). NOT retried, for two
distinct reasons: a **commit-phase** failure is ambiguous — a re-run could flip
``True``→``False`` and strand; a **fresh-conn** failure is a real DB error —
reconnecting buys nothing.
"""
@pytest.fixture
def sleeps(self, monkeypatch):
calls: list[float] = []
monkeypatch.setattr(pg_barrier.time, "sleep", calls.append)
return calls
@pytest.mark.parametrize(
"exc_cls",
[psycopg2.OperationalError, psycopg2.InterfaceError],
ids=["OperationalError", "InterfaceError"],
)
def test_execute_phase_reaped_cached_conn_retries_once(
self, claim, exc_cls, _clean_local, monkeypatch, caplog, sleeps
):
# Both _CONN_DEAD_ERRORS symptoms of an idle reap must be recognised and
# retried: OperationalError ("server closed …") and InterfaceError
# ("connection already closed") — libpq surfaces the broken socket either
# way depending on timing (mirrors TestDecrementPhaseSplitRetry).
dead = _FakeConn(execute_error=exc_cls("connection already closed"))
healthy = _FakeConn()
pg_barrier._local.conn = dead # cached → "reused"
monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: healthy)
with caplog.at_level("WARNING"):
assert claim() is True # authoritative answer taken from the retry
assert dead.executes == 1 and healthy.executes == 1 # exactly one extra try
assert dead.commits == 0 # the reaped attempt never committed
assert dead.closed is True # stale conn discarded
assert healthy.commits == 1 # committed exactly once, on the retry
assert pg_barrier._local.conn is healthy
assert sleeps == [pg_barrier._BARRIER_RETRY_BACKOFF_SECONDS]
assert "execute failed on a cached connection" in caplog.text
def test_commit_phase_failure_is_not_retried(
self, claim, _clean_local, monkeypatch, sleeps
):
# Ambiguous commit (server may have committed) → NOT retried; a re-run could
# flip the answer and strand. Propagates; no reconnect.
conn = _FakeConn(
commit_error=psycopg2.OperationalError("server closed during commit")
)
pg_barrier._local.conn = conn
monkeypatch.setattr(
pg_barrier,
"create_pg_connection",
lambda **_k: pytest.fail("must not reconnect on a commit-phase failure"),
)
with pytest.raises(psycopg2.OperationalError):
claim()
assert conn.executes == 1 and conn.commits == 1 # tried once, no retry
assert sleeps == [] # no backoff → no retry
def test_fresh_conn_execute_failure_is_not_retried(
self, claim, _clean_local, monkeypatch, sleeps
):
# A freshly-created connection failing is a real DB error, not an idle reap
# (reused is False when _local.conn starts empty) → not retried.
dead = _FakeConn(execute_error=psycopg2.OperationalError("connection refused"))
pg_barrier._local.conn = None # → reused False
monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: dead)
with pytest.raises(psycopg2.OperationalError):
claim()
assert dead.executes == 1 # no retry
assert sleeps == []
def test_second_conn_dead_failure_propagates(
self, claim, _clean_local, monkeypatch, sleeps
):
# Retry is bounded to ONE: a cached reap retries onto a fresh conn that also
# dies → propagate (don't loop, don't hit the AssertionError fall-through).
dead1 = _FakeConn(execute_error=psycopg2.OperationalError("reaped"))
dead2 = _FakeConn(execute_error=psycopg2.OperationalError("still down"))
pg_barrier._local.conn = dead1 # cached → reused
monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: dead2)
with pytest.raises(psycopg2.OperationalError):
claim()
assert dead1.executes == 1 and dead2.executes == 1 # exactly one retry
assert sleeps == [pg_barrier._BARRIER_RETRY_BACKOFF_SECONDS]
def test_non_connection_execute_error_on_reused_conn_not_retried(
self, claim, _clean_local, monkeypatch, sleeps
):
# The `conn_dead` conjunct is the only guard against retrying a real SQL
# error on a cached conn — a regression dropping it would silently re-run a
# DataError. Pin: reused conn, non-conn error → NOT retried.
bad = _FakeConn(execute_error=psycopg2.DataError("invalid byte 0x00"))
pg_barrier._local.conn = bad # cached → reused True
monkeypatch.setattr(
pg_barrier,
"create_pg_connection",
lambda **_k: pytest.fail("must not reconnect on a non-connection error"),
)
with pytest.raises(psycopg2.DataError):
claim()
assert bad.executes == 1 # no retry
assert sleeps == []
def test_retry_returns_the_lost_answer_authoritatively(
self, claim, _clean_local, monkeypatch, sleeps
):
# The "win/LOST answer is authoritative" half: after a reaped-conn retry, a
# fresh conn whose RETURNING yields no row → claim already held → returns
# False (not the truthy MagicMock default the happy-path test can only see).
dead = _FakeConn(execute_error=psycopg2.OperationalError("reaped"))
healthy = _FakeConn(fetchone_result=None) # ON CONFLICT DO NOTHING: no row
pg_barrier._local.conn = dead
monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: healthy)
assert claim() is False # authoritative "lost" from the retry
assert healthy.commits == 1
class TestReleaseOrchestrationClaimRetry:
"""``release_orchestration_claim`` is a first-write-after-idle on the failure
path whose raise the caller swallows — an un-retried idle-reap would leave the
claim committed and suppress every redelivery. Its DELETE is idempotent, so it
reuses the retry-once helper.
"""
@pytest.fixture(autouse=True)
def _no_sleep(self, monkeypatch):
monkeypatch.setattr(pg_barrier.time, "sleep", lambda *_a, **_k: None)
def test_retries_once_on_reaped_cached_conn(self, _clean_local, monkeypatch, caplog):
dead = _FakeConn(
execute_error=psycopg2.InterfaceError("connection already closed")
)
healthy = _FakeConn()
pg_barrier._local.conn = dead
monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: healthy)
with caplog.at_level("WARNING"):
pg_barrier.release_orchestration_claim("exec-1")
assert dead.executes == 1 and healthy.executes == 1 # exactly one retry
assert dead.closed is True # stale conn discarded
assert healthy.commits == 1 # committed on the retry
assert "release orchestration claim" in caplog.text
def test_non_connection_error_surfaces(self, _clean_local, monkeypatch):
# A real (non-connection) error must not be silently retried away — it
# propagates so the best-effort caller logs it.
pg_barrier._local.conn = _FakeConn(execute_error=ValueError("logic"))
monkeypatch.setattr(
pg_barrier, "create_pg_connection", lambda **_k: pytest.fail("no reconnect")
)
with pytest.raises(ValueError, match="logic"):
pg_barrier.release_orchestration_claim("exec-1")
# --- Layer 2: enqueue + link/abort with a real injected connection ---
@pytest.fixture
def barrier_db():
"""Inject a real autocommit connection into pg_barrier's thread-local.
Autocommit so each statement commits independently (like the worker tasks in
production) and the barrier's ``_cursor`` commit() is a no-op. The link/abort
tasks run in this same (test) thread, so they see the injected connection.
"""
os.environ.setdefault("TEST_DB_HOST", "127.0.0.1")
try:
conn = create_pg_connection(env_prefix="TEST_DB_")
except psycopg2.OperationalError as exc:
pytest.skip(f"Postgres not reachable: {exc}")
conn.autocommit = True
with conn.cursor() as cur:
cur.execute("SELECT to_regclass('pg_barrier_state')")
if cur.fetchone()[0] is None:
conn.close()
pytest.skip("pg_barrier_state migration not applied (run backend migrate)")
cur.execute("DELETE FROM pg_barrier_state")
pg_barrier._local.conn = conn
yield conn
with conn.cursor() as cur:
cur.execute("DELETE FROM pg_barrier_state")
conn.close()
pg_barrier._local.conn = None
def _row(conn, execution_id):
with conn.cursor() as cur:
cur.execute(
"SELECT remaining, results FROM pg_barrier_state WHERE execution_id = %s",
(execution_id,),
)
return cur.fetchone()
def _org(conn, execution_id):
with conn.cursor() as cur:
cur.execute(
"SELECT organization_id FROM pg_barrier_state WHERE execution_id = %s",
(execution_id,),
)
row = cur.fetchone()
return row[0] if row else None
def _expires_in_seconds(conn, execution_id):
"""Seconds from *now* until the barrier's expires_at (DB-side now(), so no
client-clock skew) — the fixed 6h absolute cap.
"""
with conn.cursor() as cur:
cur.execute(
"SELECT EXTRACT(EPOCH FROM (expires_at - now())) "
"FROM pg_barrier_state WHERE execution_id = %s",
(execution_id,),
)
return float(cur.fetchone()[0])
def _last_progress_age_seconds(conn, execution_id):
"""How long ago (DB-side) the barrier last made progress — the reaper's stuck
signal (UN-3661). Small = fresh; large = stalled.
"""
with conn.cursor() as cur:
cur.execute(
"SELECT EXTRACT(EPOCH FROM (now() - last_progress_at)) "
"FROM pg_barrier_state WHERE execution_id = %s",
(execution_id,),
)
return float(cur.fetchone()[0])
class TestPgBarrierEnqueue:
def test_upsert_creates_row_and_dispatches_headers(self, barrier_db):
tasks = [_mock_header_task() for _ in range(3)]
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
handle = PgBarrier().enqueue(
[t for t, _ in tasks],
callback_task_name="cb",
callback_kwargs={"execution_id": "exec-A"},
callback_queue="general",
app_instance=None,
)
assert handle.id == "exec-A"
assert _row(barrier_db, "exec-A") == (3, [])
# One PG dispatch per header, each carrying its barrier context. There is
# no .link/.link_error to assert on any more — the consumer decrements
# in-body from the injected _barrier_context.
assert mock_dispatch.call_count == 3
for i, call in enumerate(mock_dispatch.call_args_list):
assert call.kwargs["kwargs"]["_barrier_context"]["batch_index"] == i
def test_callback_descriptor_still_carries_legacy_transport(self, barrier_db):
# Rolling-deploy shim (UN-4078): a pre-UN-4078 file-processing pod reads
# callback_descriptor["transport"] and fires the callback via Celery when
# it is absent. Must stay written until the release after UN-4078.
task, _ = _mock_header_task()
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
PgBarrier().enqueue(
[task],
callback_task_name="cb",
callback_kwargs={"execution_id": "exec-LT"},
callback_queue="general",
app_instance=None,
)
ctx = mock_dispatch.call_args.kwargs["kwargs"]["_barrier_context"]
assert ctx["callback_descriptor"]["transport"] == "pg_queue"
def test_enqueue_sets_expires_cap_and_fresh_progress(self, barrier_db, monkeypatch):
# enqueue stamps expires_at = now()+ttl (the absolute cap) AND
# last_progress_at = now() (fresh), so a just-enqueued barrier is neither
# expired nor stale. (UN-3661)
monkeypatch.setenv("WORKER_BARRIER_KEY_TTL_SECONDS", "600")
task, _ = _mock_header_task()
with patch("queue_backend.dispatch.dispatch"):
PgBarrier().enqueue(
[task],
callback_task_name="cb",
callback_kwargs={"execution_id": "exec-SD"},
callback_queue="general",
app_instance=None,
)
assert 590 <= _expires_in_seconds(barrier_db, "exec-SD") <= 600 # ~ttl cap
assert _last_progress_age_seconds(barrier_db, "exec-SD") < 5 # fresh
def test_enqueue_self_heals_on_stale_connection(self, barrier_db, monkeypatch):
# End-to-end through enqueue(): the first barrier write hits a dead cached
# conn; the fix must reconnect to the REAL db, land the row exactly once,
# and still dispatch every header exactly once (no double-dispatch). This
# pins the wiring — reverting enqueue() to the inline `with _cursor()`
# would make this fail (the no-DB retry tests alone would not catch that).
monkeypatch.setattr(pg_barrier.time, "sleep", lambda *_a, **_k: None)
dead = _FakeConn(
execute_error=psycopg2.OperationalError(
"server closed the connection unexpectedly"
)
)
pg_barrier._local.conn = (
dead # the barrier_db fixture's real conn is the reconnect target
)
monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: barrier_db)
tasks = [_mock_header_task() for _ in range(3)]
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
handle = PgBarrier().enqueue(
[t for t, _ in tasks],
callback_task_name="cb",
callback_kwargs={"execution_id": "exec-HEAL"},
callback_queue="general",
app_instance=None,
)
assert handle.id == "exec-HEAL"
assert dead.closed is True # stale conn discarded by _cursor
assert _row(barrier_db, "exec-HEAL") == (3, []) # row landed once, remaining=N
assert mock_dispatch.call_count == 3 # every header dispatched exactly once
def test_fairness_forwarded_to_dispatch(self, barrier_db):
# Fairness reaches the queue row as a FairnessKey on the dispatch, not as
# an AMQP header on a cloned signature (that was the .link path).
task, _ = _mock_header_task()
fairness = FairnessKey(org_id="o", workload_type=WorkloadType.API)
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
PgBarrier().enqueue(
[task],
callback_task_name="cb",
callback_kwargs={"execution_id": "exec-F"},
callback_queue="general",
app_instance=None,
fairness=fairness,
)
assert mock_dispatch.call_args.kwargs["fairness"] is fairness
def test_upsert_overwrites_stale_state(self, barrier_db):
# A prior run left a row at remaining=1 with results; the new enqueue
# resets it.
with barrier_db.cursor() as cur:
cur.execute(
"INSERT INTO pg_barrier_state "
"(execution_id, organization_id, remaining, results, "
" created_at, expires_at, last_progress_at) "
"VALUES ('exec-R', '', 1, '[1,2]'::jsonb, now(), "
" now() + interval '1h', now())"
)
task, _ = _mock_header_task()
with patch("queue_backend.dispatch.dispatch"):
PgBarrier().enqueue(
[task, _mock_header_task()[0]],
callback_task_name="cb",
callback_kwargs={"execution_id": "exec-R"},
callback_queue="general",
app_instance=None,
)
assert _row(barrier_db, "exec-R") == (2, [])
def test_enqueue_stamps_organization_id(self, barrier_db):
# The whole reason the org column + migration exist (reaper recovery).
with patch("queue_backend.dispatch.dispatch"):
PgBarrier().enqueue(
[_mock_header_task()[0]],
callback_task_name="cb",
callback_kwargs={"execution_id": "exec-ORG", "organization_id": "org-42"},
callback_queue="general",
app_instance=None,
)
assert _org(barrier_db, "exec-ORG") == "org-42"
def test_enqueue_defaults_org_to_empty_when_absent(self, barrier_db):
with patch("queue_backend.dispatch.dispatch"):
PgBarrier().enqueue(
[_mock_header_task()[0]],
callback_task_name="cb",
callback_kwargs={"execution_id": "exec-NOORG"}, # no organization_id
callback_queue="general",
app_instance=None,
)
assert _org(barrier_db, "exec-NOORG") == ""
def test_upsert_refreshes_org_on_reenqueue(self, barrier_db):
# Exercises the ON CONFLICT DO UPDATE SET organization_id clause.
with barrier_db.cursor() as cur:
cur.execute(
"INSERT INTO pg_barrier_state "
"(execution_id, organization_id, remaining, results, "
" created_at, expires_at, last_progress_at) "
"VALUES ('exec-REORG', 'old-org', 1, '[]'::jsonb, now(), "
" now() + interval '1h', now())"
)
with patch("queue_backend.dispatch.dispatch"):
PgBarrier().enqueue(
[_mock_header_task()[0]],
callback_task_name="cb",
callback_kwargs={"execution_id": "exec-REORG", "organization_id": "new-org"},
callback_queue="general",
app_instance=None,
)
assert _org(barrier_db, "exec-REORG") == "new-org"
def test_mid_loop_dispatch_failure_deletes_row(self, barrier_db):
good, _ = _mock_header_task()
bad, _ = _mock_header_task()
barrier = PgBarrier()
with patch(
"queue_backend.dispatch.dispatch",
side_effect=[None, RuntimeError("queue write failed")],
):
with pytest.raises(RuntimeError):
barrier.enqueue(
[good, bad],
callback_task_name="cb",
callback_kwargs={"execution_id": "exec-D"},
callback_queue="general",
app_instance=None,
)
assert _row(barrier_db, "exec-D") is None # cleaned up
def _seed(conn, execution_id, remaining, *, results="[]"):
with conn.cursor() as cur:
cur.execute(
"INSERT INTO pg_barrier_state "
"(execution_id, organization_id, remaining, results, "
" created_at, expires_at, last_progress_at) "
"VALUES (%s, '', %s, %s::jsonb, now(), now() + interval '1h', now())",
(execution_id, remaining, results),
)
class TestDecrAndCheck:
def test_pending_decrements_only(self, barrier_db):
_seed(barrier_db, "exec-P", 3)
out = _barrier_pg_decrement(
{"f": 1}, execution_id="exec-P", callback_descriptor=_CALLBACK
)
assert out["status"] == "pending"
remaining, results = _row(barrier_db, "exec-P")
assert remaining == 2
assert results == [{"f": 1}]
def test_decrement_refreshes_last_progress_at(self, barrier_db):
# Each decrement re-stamps last_progress_at = now(), so a barrier that IS
# making progress never goes stale (the reaper only reaps a stalled one).
# Seed a STALE last_progress_at (as if about to be reaped) → decrement must
# refresh it. (UN-3661)
with barrier_db.cursor() as cur:
cur.execute(
"INSERT INTO pg_barrier_state "
"(execution_id, organization_id, remaining, results, "
" created_at, expires_at, last_progress_at) "
"VALUES ('exec-SL', '', 2, '[]'::jsonb, "
" now() - interval '1 hour', now() + interval '5 hours', "
" now() - interval '1 hour')"
)
assert _last_progress_age_seconds(barrier_db, "exec-SL") > 3000 # ~1h stale
_barrier_pg_decrement(
{"f": 1}, execution_id="exec-SL", callback_descriptor=_CALLBACK
)
assert _last_progress_age_seconds(barrier_db, "exec-SL") < 5 # refreshed
def test_idle_reaped_conn_self_heals_and_decrements_exactly_once(
self, barrier_db, monkeypatch
):
# End-to-end through the production entry (_barrier_pg_decrement): the
# decrement's cached conn was idle-reaped and fails the first execute; the
# phase-split retry reconnects to the REAL db and lands the decrement
# EXACTLY once (2 → 1, not 2 → 0, and one result appended). Reverting
# _apply_decrement to a plain `with _cursor()` makes this fail.
monkeypatch.setattr(pg_barrier.time, "sleep", lambda *_a, **_k: None)
_seed(barrier_db, "exec-HEALD", 2)
dead = _FakeConn(
execute_error=psycopg2.OperationalError(
"server closed the connection unexpectedly"
)
)
pg_barrier._local.conn = dead # the barrier_db fixture conn is the target
monkeypatch.setattr(pg_barrier, "create_pg_connection", lambda **_k: barrier_db)
out = _barrier_pg_decrement(
{"f": "x"}, execution_id="exec-HEALD", callback_descriptor=_CALLBACK
)
assert out["status"] == "pending" and out["remaining"] == 1
assert dead.closed is True # stale conn discarded
assert _row(barrier_db, "exec-HEALD") == (1, [{"f": "x"}]) # decremented once
def test_pg_terminate_backend_mid_decrement_fires_callback_once(
self, barrier_db, monkeypatch
):
# The strongest pin: a REAL server-side connection kill (the
# pg_terminate_backend analog of a PgBouncer idle-reap) mid-decrement. The
# victim is a genuine non-autocommit connection (production posture); we
# terminate its backend for real, then drive the decrement that takes the
# barrier to 0. The phase-split retry must reconnect to live PG, land the
# decrement EXACTLY once, and fire the callback EXACTLY once. A
# double-count would drive remaining to -1 → "abandoned" → no callback, so
# asserting "complete" + one signature call + row deleted proves
# exactly-once against a real terminated socket.
monkeypatch.setattr(pg_barrier.time, "sleep", lambda *_a, **_k: None)
_seed(barrier_db, "exec-PTB", 1) # last batch: decrement → 0 fires callback
victim = create_pg_connection(env_prefix="TEST_DB_") # non-autocommit
with victim.cursor() as cur:
cur.execute("SELECT pg_backend_pid()")
victim_pid = cur.fetchone()[0]
victim.rollback() # return to IDLE so the decrement entry-guard passes
pg_barrier._local.conn = victim # the decrement uses this cached conn
# Kill the victim's backend from the admin (fixture) connection — the
# client still thinks it's open (conn.closed == 0), exactly like an idle
# reap; the failure surfaces on the next statement (the decrement UPDATE).
with barrier_db.cursor() as cur:
cur.execute("SELECT pg_terminate_backend(%s)", (victim_pid,))
# The retry reconnects via create_pg_connection; in tests that must target
# the reachable TEST_DB_ (the bare DB_* host is the in-container name).
monkeypatch.setattr(
pg_barrier,
"create_pg_connection",
lambda **_k: create_pg_connection(env_prefix="TEST_DB_"),
)
with patch("queue_backend.dispatch.dispatch") as sig:
sig.return_value = MagicMock(id="cb-ptb")
out = _barrier_pg_decrement(
{"f": "x"}, execution_id="exec-PTB", callback_descriptor=_CALLBACK
)
assert out["status"] == "complete" # reached 0 exactly once (not -1)
sig.assert_called_once() # callback fired exactly once
assert _row(barrier_db, "exec-PTB") is None # barrier torn down after fire
assert victim.closed # the terminated conn was discarded by the retry
def test_complete_fires_callback_with_aggregated_results(self, barrier_db):
_seed(barrier_db, "exec-C", 1, results='[{"f": "a"}]')
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
mock_dispatch.return_value = MagicMock(id="cb-task-1")
out = _barrier_pg_decrement(
{"f": "b"}, execution_id="exec-C", callback_descriptor=_CALLBACK
)
assert out["status"] == "complete"
# Callback got the full aggregated list as its first positional arg.
assert mock_dispatch.call_args.kwargs["args"] == [[{"f": "a"}, {"f": "b"}]]
assert _row(barrier_db, "exec-C") is None # row deleted after dispatch
def test_complete_path_passes_fairness_header(self, barrier_db):
_seed(barrier_db, "exec-FH", 1)
# A full header payload: the PG path *parses* it back into a FairnessKey
# (the Celery path only forwarded the dict opaquely), so a partial one
# would fail on the missing workload_type rather than assert anything.
descriptor = {
**_CALLBACK,
"fairness_headers": {
FAIRNESS_HEADER_NAME: {
"org_id": "o",
"workload_type": "api",
"pipeline_priority": 5,
}
},
}
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
mock_dispatch.return_value = MagicMock(id="cb")
_barrier_pg_decrement(
{"f": 1}, execution_id="exec-FH", callback_descriptor=descriptor
)
# Reconstructed into a FairnessKey and handed to the dispatch, so the
# callback rides the same org/priority its header tasks did.
forwarded = mock_dispatch.call_args.kwargs["fairness"]
assert forwarded.org_id == "o"
assert forwarded.workload_type is WorkloadType.API
assert forwarded.pipeline_priority == 5
def test_callback_dispatch_failure_preserves_row(self, barrier_db):
# The central failure-masking invariant: dispatch happens BEFORE the row
# is deleted, so a callback-dispatch failure leaves the row in place
# (reclaimed by expiry) — guards against a delete-before-dispatch reorder.
_seed(barrier_db, "exec-CF", 1)
with patch(
"queue_backend.dispatch.dispatch",
side_effect=RuntimeError("queue write failed"),
):
with pytest.raises(RuntimeError):
_barrier_pg_decrement(
{"f": 1}, execution_id="exec-CF", callback_descriptor=_CALLBACK
)
assert _row(barrier_db, "exec-CF") is not None # row survives for TTL reclaim
def test_list_result_appended_as_single_element(self, barrier_db):
# jsonb_build_array() must append a list-shaped result as ONE element
# (plain `||` would concatenate it). Guards that choice.
_seed(barrier_db, "exec-L", 2)
_barrier_pg_decrement(
[1, 2], execution_id="exec-L", callback_descriptor=_CALLBACK
)
remaining, results = _row(barrier_db, "exec-L")
assert remaining == 1
assert results == [[1, 2]] # one element that is the list, not [1, 2]
def test_nul_byte_result_is_repaired_and_decrements(self, barrier_db):
# UN-4126: a NUL survives json.dumps but jsonb rejects it. It is now
# stripped by dumps_for_jsonb rather than costing the barrier — a stray
# control byte from native_text extraction is not a reason to fail an
# execution. Seeded at 2 so this asserts the decrement, not the callback.
_seed(barrier_db, "exec-NB", 2)
out = _barrier_pg_decrement(
{"f": "bad\x00value"},
execution_id="exec-NB",
callback_descriptor=_CALLBACK,
)
assert out["status"] == "pending"
remaining, results = _row(barrier_db, "exec-NB")
assert remaining == 1
assert results == [{"f": "badvalue"}] # NUL gone, everything else intact
def test_unencodable_result_raises_and_leaves_the_row_for_the_caller(
self, barrier_db
):
# NaN is deliberately NOT repaired (null/0 would corrupt the value), so it
# fails at the encoder rather than at the jsonb cast. The decrement must
# propagate WITHOUT releasing the barrier row: that row is the reaper's
# only recovery handle, and run_batch_with_barrier releases it only after
# confirming the execution is terminal. Tearing down here would strand the
# execution permanently whenever that confirmation fails.
_seed(barrier_db, "exec-NaN", 1)
with pytest.raises(ValueError):
_barrier_pg_decrement(
{"f": float("nan")},
execution_id="exec-NaN",
callback_descriptor=_CALLBACK,
)
assert _row(barrier_db, "exec-NaN") == (1, []) # intact for the caller
def test_decrement_after_abort_does_not_fire(self, barrier_db):
# The new failure-masking model: an aborted barrier is GONE (abort
# deletes the row), so a late in-flight decrement finds no row and never
# fires — even when it would otherwise have hit remaining == 0.
_seed(barrier_db, "exec-DA", 1)
barrier_pg_abort(execution_id="exec-DA") # header failed → row deleted
with patch("queue_backend.dispatch.dispatch") as sig:
out = _barrier_pg_decrement(
{"f": 1}, execution_id="exec-DA", callback_descriptor=_CALLBACK
)
assert out["status"] == "abandoned"
sig.assert_not_called()
def test_negative_remaining_does_not_fire(self, barrier_db):
_seed(barrier_db, "exec-N", 0) # decrement → -1
with patch("queue_backend.dispatch.dispatch") as sig:
out = _barrier_pg_decrement(
{"f": 1}, execution_id="exec-N", callback_descriptor=_CALLBACK
)
assert out["status"] == "abandoned"
sig.assert_not_called()
assert _row(barrier_db, "exec-N") is None
def test_missing_row_does_not_fire(self, barrier_db):
with patch("queue_backend.dispatch.dispatch") as sig:
out = _barrier_pg_decrement(
{"f": 1}, execution_id="nope", callback_descriptor=_CALLBACK
)
assert out["status"] == "abandoned"
sig.assert_not_called()
def test_unserialisable_result_raises(self, barrier_db):
_seed(barrier_db, "exec-U", 1)
bad_result = {object()}
with pytest.raises(TypeError):
_barrier_pg_decrement(
bad_result, execution_id="exec-U", callback_descriptor=_CALLBACK
)
class TestDecrementCoreExtraction:
"""The decrement logic lives in a plain ``_barrier_pg_decrement`` core so the
9e PR 2c PG-consumed path can call it in-body (a PG-consumed task fires no
``.link``). The Celery ``@worker_task`` is a thin delegator; both paths must
share one implementation (no drift). (Inert in 2a — no PG caller yet.)
"""
def test_core_is_a_plain_callable_not_a_celery_task(self):
# A ``@worker_task`` exposes ``.name`` / ``.delay``; the in-body core
# must not, or callers could accidentally re-dispatch instead of running.
assert not hasattr(_barrier_pg_decrement, "name")
assert not hasattr(_barrier_pg_decrement, "delay")
def test_core_runs_the_decrement_in_body(self, barrier_db):
# Called directly (no Celery), the core performs the atomic decrement.
_seed(barrier_db, "exec-core", 3)
out = _barrier_pg_decrement(
{"f": 1}, execution_id="exec-core", callback_descriptor=_CALLBACK
)
assert out["status"] == "pending"
remaining, results = _row(barrier_db, "exec-core")
assert remaining == 2
assert results == [{"f": 1}]
def test_worker_task_produces_same_decrement_as_core(self, barrier_db):
# The real no-drift guard: the wrapper, run against a real seeded row,
# must produce the SAME observable decrement as a direct core call
# (mirror of test_core_runs_the_decrement_in_body). Unlike the mocked
# forwarding test below, this would catch a renamed/reordered core param
# — the core is NOT mocked away.
_seed(barrier_db, "exec-wrap", 3)
out = _barrier_pg_decrement(
{"f": 1}, execution_id="exec-wrap", callback_descriptor=_CALLBACK
)
assert out["status"] == "pending"
remaining, results = _row(barrier_db, "exec-wrap")
assert remaining == 2
assert results == [{"f": 1}]
# ``test_worker_task_forwards_kwargs_verbatim`` is gone with the
# ``barrier_pg_decr_and_check`` @worker_task wrapper it pinned: there is no
# wrapper left to forward anything, only the core called directly in-body.
def test_open_transaction_on_shared_conn_raises(self):
# The in-body contract is enforced loudly: a caller that enters with an
# already-open transaction on the shared connection is rejected before
# any decrement runs (would otherwise corrupt 'remaining' and hang the
# barrier to expiry). Guards the 2c in-body caller.
import psycopg2.extensions
conn = MagicMock()
conn.get_transaction_status.return_value = (
psycopg2.extensions.TRANSACTION_STATUS_INTRANS
)
with patch.object(pg_barrier, "_get_conn", return_value=conn):
with pytest.raises(RuntimeError, match="own committed transaction"):
_barrier_pg_decrement(
{"f": 1}, execution_id="exec-tx", callback_descriptor=_CALLBACK
)
conn.cursor.assert_not_called() # rejected before touching the DB
class TestAbort:
def test_claims_and_deletes(self, barrier_db):
_seed(barrier_db, "exec-X", 2)
out = barrier_pg_abort(execution_id="exec-X")
assert out["status"] == "aborted"
assert _row(barrier_db, "exec-X") is None
def test_concurrent_aborts_deduplicate(self, barrier_db):
_seed(barrier_db, "exec-Y", 2)
first = barrier_pg_abort(execution_id="exec-Y")
second = barrier_pg_abort(execution_id="exec-Y")
assert first["status"] == "aborted"
assert second["status"] == "already_aborted" # row gone / claim lost
def test_registered_under_canonical_name(self):
assert barrier_pg_abort.name == "barrier_pg_abort"
def test_max_retries_zero(self):
# A retry would replay the teardown. (The decrement's no-replay contract
# is enforced by its in-body caller, not a task option — its @worker_task
# wrapper went with the Celery .link path.)
assert barrier_pg_abort.max_retries == 0
# --- Layer 3: atomicity through the real task + DB constraint ---
class TestDecrementAtomicityThroughTask:
def test_exactly_one_task_fires_the_callback(self, barrier_db):
# Two header-task links decrement a remaining=2 barrier concurrently,
# each through barrier_pg_decr_and_check itself (not raw SQL). The row
# lock serialises them so exactly one returns "complete" and the callback
# is dispatched exactly once — the real single-fire guarantee.
import threading
_seed(barrier_db, "exec-Z", 2)
statuses: dict[str, str] = {}
conns = []
def run(label):
conn = create_pg_connection(env_prefix="TEST_DB_")
conn.autocommit = True
conns.append(conn)
pg_barrier._local.conn = conn # this thread's own connection
try:
out = _barrier_pg_decrement(
{"t": label}, execution_id="exec-Z", callback_descriptor=_CALLBACK
)
statuses[label] = out["status"]
finally:
pg_barrier._local.conn = None
with patch("queue_backend.dispatch.dispatch") as sig:
sig.return_value = MagicMock(id="cb")
threads = [threading.Thread(target=run, args=(x,)) for x in ("a", "b")]
for t in threads:
t.start()
for t in threads:
t.join(timeout=10)
assert all(not t.is_alive() for t in threads)
assert sorted(statuses.values()) == ["complete", "pending"]
assert sig.call_count == 1 # single fire
for c in conns:
c.close()
class TestDbConstraint:
def test_expires_at_must_exceed_created_at(self, barrier_db):
# The one writer-proof invariant (workers SQL can't import the model).
with barrier_db.cursor() as cur:
with pytest.raises(psycopg2.errors.CheckViolation):
cur.execute(
"INSERT INTO pg_barrier_state "
"(execution_id, organization_id, remaining, results, "
" created_at, expires_at, last_progress_at) "
"VALUES ('bad', '', 1, '[]'::jsonb, now(), now(), now())" # expires==created
)
class TestPgFireAndForgetMode:
"""9e PR 2c — PgBarrier's ``transport="pg_queue"`` fire-and-forget path:
headers dispatched onto PG (no ``.link``), in-body claim + decrement, callback
self-chained onto PG, and dedup-marker cleanup at enqueue/finalise/abort.
"""
def test_enqueue_pg_mode_dispatches_headers_via_pg_with_context(self, barrier_db):
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
# A header with a pre-existing kwarg, real args, and a queue — to prove
# the Signature→dispatch unpacking preserves all three (a dropped args or
# queue would silently process an empty/misrouted batch).
h0 = _pg_header(args=[{"file": "f0"}], queue="api_file_processing")
h0.kwargs = {"pre_existing": "keep"}
h1 = _pg_header(args=[{"file": "f1"}], queue="api_file_processing")
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
PgBarrier().enqueue(
[h0, h1],
callback_task_name="process_batch_callback",
callback_kwargs={"execution_id": "exec-pg"},
callback_queue="general",
app_instance=None,
)
assert mock_dispatch.call_count == 2 # one per header, no .link
for i, call in enumerate(mock_dispatch.call_args_list):
assert call.kwargs["backend"] is QueueBackend.PG
assert call.kwargs["args"] == [{"file": f"f{i}"}] # args preserved
assert call.kwargs["queue"] == "api_file_processing" # queue preserved
ctx = call.kwargs["kwargs"]["_barrier_context"]
assert ctx["execution_id"] == "exec-pg"
assert ctx["batch_index"] == i
# The pre-existing kwarg on h0 survives alongside the injected context.
assert mock_dispatch.call_args_list[0].kwargs["kwargs"]["pre_existing"] == "keep"
def test_enqueue_pg_mode_clears_stale_dedup_on_reuse(self, barrier_db):
# greptile #2068: a re-enqueue with the same execution_id must wipe prior
# dedup markers atomically with the barrier reset, else every claim_batch
# returns False and the barrier hangs.
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
claim_batch("exec-reuse", 0)
claim_batch("exec-reuse", 1)
with patch("queue_backend.dispatch.dispatch"):
PgBarrier().enqueue(
[_pg_header()],
callback_task_name="process_batch_callback",
callback_kwargs={"execution_id": "exec-reuse"},
callback_queue="general",
app_instance=None,
)
with barrier_db.cursor() as cur:
cur.execute(
"SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s",
("exec-reuse",),
)
assert cur.fetchone()[0] == 0 # stale markers wiped by the UPSERT block
def test_run_batch_with_barrier_first_delivery_runs_and_decrements(self, barrier_db):
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
_seed(barrier_db, "exec-fd", 2)
ctx = {
"execution_id": "exec-fd",
"batch_index": 0,
"callback_descriptor": _CALLBACK,
}
work = MagicMock(return_value={"ok": 1})
out = run_batch_with_barrier(ctx, work)
work.assert_called_once()
assert out == {"ok": 1}
remaining, results = _row(barrier_db, "exec-fd")
assert remaining == 1
assert results == [{"ok": 1}]
def test_run_batch_with_barrier_redelivery_skips_work_and_decrement(self, barrier_db):
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
_seed(barrier_db, "exec-re", 2)
claim_batch("exec-re", 0) # batch already claimed → this is a redelivery
ctx = {
"execution_id": "exec-re",
"batch_index": 0,
"callback_descriptor": _CALLBACK,
}
work = MagicMock(return_value={"ok": 1})
out = run_batch_with_barrier(ctx, work)
work.assert_not_called() # no reprocessing
assert out["status"] == "skipped_redelivery"
remaining, _ = _row(barrier_db, "exec-re")
assert remaining == 2 # NOT decremented again
def _dedup_count(self, conn, execution_id):
with conn.cursor() as cur:
cur.execute(
"SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s",
(execution_id,),
)
return cur.fetchone()[0]
def test_exception_marks_error_then_tears_down_keeping_markers(
self, barrier_db, monkeypatch
):
# Confirmed ERROR mark → safe to tear the barrier row down, but KEEP the
# per-batch dedup marker so this message's redelivery is skipped by
# claim_batch (not re-run wholesale).
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
_seed(barrier_db, "exec-err", 2)
marked = []
monkeypatch.setattr(
pg_barrier,
"_mark_execution_error_on_abort",
lambda ctx, *, reason: marked.append(reason) or True,
)
ctx = {
"execution_id": "exec-err",
"batch_index": 0,
"callback_descriptor": _CALLBACK,
}
def boom():
raise RuntimeError("batch failed")
with pytest.raises(RuntimeError, match="batch failed"):
run_batch_with_barrier(ctx, boom)
assert marked # execution marked ERROR before teardown
# Row torn down (mirror chord abort) but the dedup marker survives.
assert _row(barrier_db, "exec-err") is None
assert self._dedup_count(barrier_db, "exec-err") == 1 # claim marker kept
def test_exception_with_unconfirmed_mark_leaves_barrier_row(
self, barrier_db, monkeypatch
):
# ERROR mark NOT confirmed (backend down / no org) → the barrier row is the
# reaper's only recovery handle, so it must be LEFT intact, not deleted.
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
_seed(barrier_db, "exec-strand", 2)
monkeypatch.setattr(
pg_barrier,
"_mark_execution_error_on_abort",
lambda ctx, *, reason: False,
)
ctx = {
"execution_id": "exec-strand",
"batch_index": 0,
"callback_descriptor": _CALLBACK,
}
def boom():
raise RuntimeError("batch failed")
with pytest.raises(RuntimeError, match="batch failed"):
run_batch_with_barrier(ctx, boom)
# Barrier row preserved for the reaper (remaining untouched).
remaining, _ = _row(barrier_db, "exec-strand")
assert remaining == 2
def test_fire_barrier_callback_pg_self_chains_via_dispatch(self):
descriptor = dict(_CALLBACK)
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
mock_dispatch.return_value = MagicMock(id="pg-cb-1")
cb_id = _fire_barrier_callback(descriptor, [{"r": 1}])
assert cb_id == "pg-cb-1"
assert mock_dispatch.call_args.kwargs["backend"] is QueueBackend.PG
assert mock_dispatch.call_args.kwargs["args"] == [[{"r": 1}]]
def test_fire_barrier_callback_pg_tags_transport_marker(self):
# The PG callback carries the _pg_transport marker so the aggregating
# callback can gate its at-least-once duplicate guard on it. The shared
# descriptor must NOT be mutated (a copy is dispatched).
descriptor = dict(_CALLBACK)
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
mock_dispatch.return_value = MagicMock(id="pg-cb")
_fire_barrier_callback(descriptor, [{"r": 1}])
dispatched = mock_dispatch.call_args.kwargs["kwargs"]
assert dispatched.get(pg_barrier.PG_TRANSPORT_CALLBACK_KWARG) is True
assert pg_barrier.PG_TRANSPORT_CALLBACK_KWARG not in descriptor["kwargs"]
def test_fire_barrier_callback_pg_carries_fairness(self):
# greptile: the PG callback must ride the producer's org/priority (parity
# with the Celery path), reconstructed from the stored fairness_headers.
descriptor = {
**_CALLBACK,
"fairness_headers": {
FAIRNESS_HEADER_NAME: {
"org_id": "org-9",
"workload_type": "api",
"pipeline_priority": 8,
}
},
}
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
mock_dispatch.return_value = MagicMock(id="pg-cb")
_fire_barrier_callback(descriptor, [{"r": 1}])
fairness = mock_dispatch.call_args.kwargs["fairness"]
assert fairness is not None
assert fairness.org_id == "org-9"
assert fairness.pipeline_priority == 8
def test_fire_barrier_callback_pg_without_fairness_passes_none(self):
# No producer key → None (dispatch writes neutral defaults), not a crash.
descriptor = dict(_CALLBACK) # fairness_headers None
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
mock_dispatch.return_value = MagicMock(id="pg-cb")
_fire_barrier_callback(descriptor, [{"r": 1}])
assert mock_dispatch.call_args.kwargs["fairness"] is None
def test_abort_clears_dedup_markers(self, barrier_db):
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
_seed(barrier_db, "exec-ab", 2)
claim_batch("exec-ab", 0)
barrier_pg_abort(execution_id="exec-ab")
with barrier_db.cursor() as cur:
cur.execute(
"SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s",
("exec-ab",),
)
assert cur.fetchone()[0] == 0
def test_abort_preserve_flag_keeps_dedup_markers(self, barrier_db):
# preserve_dedup_markers=True (the in-body PG path): the barrier row is
# still deleted, but the marker survives so a redelivered batch is skipped
# by claim_batch rather than re-run wholesale.
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
_seed(barrier_db, "exec-keep", 2)
claim_batch("exec-keep", 0)
barrier_pg_abort(execution_id="exec-keep", preserve_dedup_markers=True)
assert _row(barrier_db, "exec-keep") is None # barrier row still deleted
with barrier_db.cursor() as cur:
cur.execute(
"SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s",
("exec-keep",),
)
assert cur.fetchone()[0] == 1 # marker preserved
assert claim_batch("exec-keep", 0) is False # redelivery would be skipped
def test_last_batch_self_chains_callback_to_pg_and_cleans_up(self, barrier_db):
# The PR's core promise, end to end: the batch that drives remaining → 0
# self-chains the aggregating callback onto PG (backend=PG, args=[results])
# and clears both the barrier row and the dedup markers.
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
_seed(barrier_db, "exec-last", 1) # this batch is the last → remaining → 0
descriptor = {
"task_name": "process_batch_callback_api",
"kwargs": {"execution_id": "exec-last"},
"queue": "api_file_processing_callback",
"fairness_headers": None,
}
ctx = {
"execution_id": "exec-last",
"batch_index": 0,
"callback_descriptor": descriptor,
}
with patch("queue_backend.dispatch.dispatch") as mock_dispatch:
mock_dispatch.return_value = MagicMock(id="pg-cb")
out = run_batch_with_barrier(ctx, lambda: {"successful_files": 1})
assert out == {"successful_files": 1}
# callback self-chained onto PG with the aggregated results
assert mock_dispatch.call_args.kwargs["backend"] is QueueBackend.PG
assert mock_dispatch.call_args.args[0] == "process_batch_callback_api"
assert mock_dispatch.call_args.kwargs["args"] == [[{"successful_files": 1}]]
# barrier row + dedup markers cleaned up at finalise
assert _row(barrier_db, "exec-last") is None
with barrier_db.cursor() as cur:
cur.execute(
"SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s",
("exec-last",),
)
assert cur.fetchone()[0] == 0
def test_decrement_failure_aborts_barrier(self, barrier_db, monkeypatch):
# #69: a decrement-side failure (after the work succeeded) must tear the
# barrier down in-body — else the dedup marker is committed, redelivery is
# blocked, and the barrier strands to expiry. With a confirmed ERROR mark
# the row is torn down (same path as a work failure).
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
_seed(barrier_db, "exec-decfail", 2)
monkeypatch.setattr(
pg_barrier, "_mark_execution_error_on_abort", lambda ctx, *, reason: True
)
ctx = {
"execution_id": "exec-decfail",
"batch_index": 0,
"callback_descriptor": _CALLBACK,
}
with patch.object(
pg_barrier, "_barrier_pg_decrement", side_effect=RuntimeError("decr boom")
):
with pytest.raises(RuntimeError, match="decr boom"):
run_batch_with_barrier(ctx, lambda: {"ok": 1})
assert _row(barrier_db, "exec-decfail") is None # barrier torn down
def test_pg_mid_loop_dispatch_failure_deletes_row_and_clears_dedup(self, barrier_db):
# The PG-branch counterpart of test_mid_loop_dispatch_failure_deletes_row:
# a header PG-dispatch failure mid fan-out deletes the barrier row AND
# reclaims dedup markers (greptile #2069) — an already-claimed earlier
# header's marker would otherwise orphan, since the in-flight abort is a
# no-op once the barrier row is gone.
#
# The marker must be claimed AFTER enqueue's UPSERT block (which wipes
# stale markers for this execution_id) — else the assertion would pass on
# the UPSERT, not the mid-loop clear under test. So the first dispatch
# call claims it (a fast consumer on the PG path), the second fails.
with barrier_db.cursor() as cur:
cur.execute("DELETE FROM pg_batch_dedup")
def dispatch_side_effect(*args, **kwargs):
if not dispatch_side_effect.claimed:
dispatch_side_effect.claimed = True
claim_batch("exec-midfail", 0) # marker created post-UPSERT
return MagicMock(id="1")
raise RuntimeError("broker down")
dispatch_side_effect.claimed = False
with patch("queue_backend.dispatch.dispatch", side_effect=dispatch_side_effect):
barrier = PgBarrier()
headers = [_pg_header(), _pg_header()]
with pytest.raises(RuntimeError, match="broker down"):
barrier.enqueue(
headers,
callback_task_name="process_batch_callback",
callback_kwargs={"execution_id": "exec-midfail"},
callback_queue="general",
app_instance=None,
)
assert _row(barrier_db, "exec-midfail") is None # row deleted on failure
with barrier_db.cursor() as cur:
cur.execute(
"SELECT count(*) FROM pg_batch_dedup WHERE execution_id = %s",
("exec-midfail",),
)
# The marker existed post-UPSERT and was removed by the mid-loop
# clear_execution_batches under test (not by the UPSERT reset).
assert cur.fetchone()[0] == 0
class TestMarkExecutionErrorOnAbort:
"""``_mark_execution_error_on_abort`` — the in-body PG failure → terminal mark
(no DB; the internal API client + mark helper are mocked).
"""
def _ctx(self, *, org):
kwargs = {"execution_id": "exec-1", "pipeline_id": "pipe-1"}
if org is not None:
kwargs["organization_id"] = org
return {
"execution_id": "exec-1",
"batch_index": 0,
"callback_descriptor": {**_CALLBACK, "kwargs": kwargs},
}
def test_no_org_returns_false_without_building_client(self, monkeypatch):
# No org → can't call the org-scoped API; must NOT build a client or mark.
built = MagicMock(side_effect=AssertionError("client built without org"))
monkeypatch.setattr("shared.api.InternalAPIClient", built)
assert (
pg_barrier._mark_execution_error_on_abort(
self._ctx(org=None), reason="batch 0 failed"
)
is False
)
def test_org_present_marks_error_and_returns_helper_result(self, monkeypatch):
client = MagicMock(name="api_client")
monkeypatch.setattr(
"shared.api.InternalAPIClient", MagicMock(return_value=client)
)
mark = MagicMock(return_value=True)
monkeypatch.setattr("queue_backend.pg_queue.recovery.mark_execution_error", mark)
out = pg_barrier._mark_execution_error_on_abort(
self._ctx(org="org-9"), reason="batch 0 failed"
)
assert out is True
mark.assert_called_once()
args, kwargs = mark.call_args
assert args[0] is client
assert args[1] == "exec-1"
assert args[2] == "org-9"
assert kwargs["error_message"] == "[pg-barrier-abort] batch 0 failed."
def test_helper_false_propagates(self, monkeypatch):
monkeypatch.setattr(
"shared.api.InternalAPIClient", MagicMock(return_value=MagicMock())
)
monkeypatch.setattr(
"queue_backend.pg_queue.recovery.mark_execution_error",
MagicMock(return_value=False),
)
assert (
pg_barrier._mark_execution_error_on_abort(self._ctx(org="org-9"), reason="x")
is False
)
def test_client_build_failure_returns_false(self, monkeypatch):
monkeypatch.setattr(
"shared.api.InternalAPIClient",
MagicMock(side_effect=RuntimeError("no config")),
)
assert (
pg_barrier._mark_execution_error_on_abort(self._ctx(org="org-9"), reason="x")
is False
)
if __name__ == "__main__":
pytest.main([__file__, "-v"])