LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Tests for :class:`queue_backend.pg_queue.leader_election.LeaderLease`.
Two layers:
1. **Env / construction** — no DB (``lease_seconds_from_env``, worker-id guard).
2. **Lease semantics** — real Postgres. Each ``LeaderLease`` is given its own
autocommit connection so two instances genuinely race the single
``pg_orchestrator_lock`` row. Skips if Postgres is unreachable or the
``pg_orchestrator_lock`` migration is unapplied.
The two load-bearing correctness properties:
- concurrent ``try_acquire`` on a free lock → **exactly one** winner;
- ``renew`` returns ``False`` after a standby took over a stale lease (the
signal that tells a stalled leader to stop acting).
"""
from __future__ import annotations
import contextlib
import os
import threading
from types import SimpleNamespace
from unittest.mock import MagicMock
import psycopg2
import pytest
from queue_backend.pg_queue.connection import create_pg_connection
from queue_backend.pg_queue.leader_election import (
LeaderLease,
default_worker_id,
lease_seconds_from_env,
)
# --- Layer 1: env + construction (no DB) ---
class TestLeaseSecondsEnv:
def test_default_is_ten_seconds(self, monkeypatch):
monkeypatch.delenv("WORKER_PG_ORCHESTRATOR_LEASE_SECONDS", raising=False)
assert lease_seconds_from_env() == 10
def test_overridable(self, monkeypatch):
monkeypatch.setenv("WORKER_PG_ORCHESTRATOR_LEASE_SECONDS", "30")
assert lease_seconds_from_env() == 30
@pytest.mark.parametrize("bad", ["0", "-5", "abc", "1.5"])
def test_invalid_raises(self, monkeypatch, bad):
monkeypatch.setenv("WORKER_PG_ORCHESTRATOR_LEASE_SECONDS", bad)
with pytest.raises(ValueError):
lease_seconds_from_env()
class TestConstruction:
@pytest.mark.parametrize("bad", ["", " "])
def test_empty_worker_id_rejected(self, bad):
conn = object()
with pytest.raises(ValueError, match="non-empty"):
LeaderLease(bad, conn=conn) # conn unused — guard fires first
def test_non_positive_lease_seconds_rejected(self):
conn = object()
with pytest.raises(ValueError):
LeaderLease("w1", lease_seconds=0, conn=conn)
def test_default_worker_id_shape_and_idempotent(self):
default_worker_id.cache_clear()
wid = default_worker_id()
# host:pid:rand shape, three colon-separated parts.
assert len(wid.split(":")) == 3
# Idempotent within a process (lru_cache) — repeated calls match, so a
# caller can pass it inline without drifting the worker id.
assert default_worker_id() == wid
default_worker_id.cache_clear()
# --- Layer 1b: connection recovery / ownership (mocked, no DB) ---
def _mock_conn(*, execute_exc=None, rollback_exc=None, fetch=(1,)):
cur = MagicMock()
if execute_exc is not None:
cur.execute.side_effect = execute_exc
cur.fetchone.return_value = fetch
cur.rowcount = 1
conn = MagicMock()
conn.closed = 0
conn.cursor.return_value.__enter__.return_value = cur
conn.cursor.return_value.__exit__.return_value = False
if rollback_exc is not None:
conn.rollback.side_effect = rollback_exc
return conn
_FACTORY_PATH = "queue_backend.pg_queue.leader_election.create_pg_connection"
class TestConnectionRecovery:
"""Owned vs injected recovery branches of ``_cursor`` — the load-bearing
self-recovery the docstrings advertise (mirrors PgQueueClient's suite)."""
def test_owned_conn_recovered_on_operational_error(self, monkeypatch):
bad = _mock_conn(execute_exc=psycopg2.OperationalError("dead"))
good = _mock_conn(fetch=(1,))
factory = MagicMock(side_effect=[bad, good])
monkeypatch.setattr(_FACTORY_PATH, factory)
lease = LeaderLease("w") # owns its connection (conn=None)
with pytest.raises(psycopg2.OperationalError):
lease.try_acquire()
bad.rollback.assert_called_once()
bad.close.assert_called_once()
assert lease._conn is None # dead owned handle dropped
assert lease.try_acquire() is True # next call reconnects to good
assert factory.call_count == 2
def test_owned_conn_recovered_when_rollback_fails(self, monkeypatch):
# Non-Operational error whose rollback also fails → still treated dead.
bad = _mock_conn(
execute_exc=RuntimeError("boom"),
rollback_exc=psycopg2.DatabaseError("socket gone"),
)
monkeypatch.setattr(_FACTORY_PATH, MagicMock(side_effect=[bad]))
lease = LeaderLease("w")
with pytest.raises(RuntimeError):
lease.try_acquire()
bad.close.assert_called_once()
assert lease._conn is None
def test_injected_conn_never_swapped_on_error(self):
# The High finding: an injected (caller-owned) connection must NOT be
# closed + silently replaced with a DB_-env one on a transient error.
bad = _mock_conn(execute_exc=psycopg2.OperationalError("dead"))
lease = LeaderLease("w", conn=bad)
with pytest.raises(psycopg2.OperationalError):
lease.try_acquire()
bad.rollback.assert_called_once()
bad.close.assert_not_called()
assert lease._conn is bad # caller's connection untouched
# --- Layer 2: lease semantics (real Postgres) ---
def _new_conn():
os.environ.setdefault("TEST_DB_HOST", "127.0.0.1")
conn = create_pg_connection(env_prefix="TEST_DB_")
conn.autocommit = True
return conn
@pytest.fixture
def lock_db():
"""Reset the single lock row to free before/after each test.
Yields a helper connection (for assertions / ageing the lease); each
``LeaderLease`` under test gets its OWN connection so concurrency is real.
Skips when Postgres is unreachable or the migration is unapplied.
"""
try:
conn = _new_conn()
except psycopg2.OperationalError as exc:
pytest.skip(f"Postgres not reachable: {exc}")
with conn.cursor() as cur:
cur.execute("SELECT to_regclass('pg_orchestrator_lock')")
if cur.fetchone()[0] is None:
conn.close()
pytest.skip(
"pg_orchestrator_lock migration not applied (run backend migrate)"
)
# Ensure the singleton row exists and is free (the migration seeds it,
# but reset defensively so tests are order-independent).
cur.execute(
"INSERT INTO pg_orchestrator_lock (id, leader, acquired_at) "
"VALUES (1, '', now()) "
"ON CONFLICT (id) DO UPDATE SET leader = '', acquired_at = now()"
)
extra_conns: list = []
def make_lease(worker_id, **kw):
c = _new_conn()
extra_conns.append(c)
return LeaderLease(worker_id, conn=c, **kw)
yield SimpleNamespace(conn=conn, make_lease=make_lease)
with conn.cursor() as cur:
cur.execute(
"UPDATE pg_orchestrator_lock SET leader = '', acquired_at = now() "
"WHERE id = 1"
)
for c in extra_conns:
with contextlib.suppress(Exception):
c.close()
conn.close()
def _leader(conn) -> str:
with conn.cursor() as cur:
cur.execute("SELECT leader FROM pg_orchestrator_lock WHERE id = 1")
return cur.fetchone()[0]
def _age_lease(conn, seconds: int) -> None:
"""Push acquired_at into the past so the lease looks stale."""
with conn.cursor() as cur:
cur.execute(
"UPDATE pg_orchestrator_lock "
"SET acquired_at = now() - make_interval(secs => %s) WHERE id = 1",
(seconds,),
)
class TestLeaderLease:
def test_acquire_on_free_lock_succeeds(self, lock_db):
lease = lock_db.make_lease("w1")
assert lease.try_acquire() is True
assert _leader(lock_db.conn) == "w1"
def test_second_acquirer_fails_while_fresh(self, lock_db):
a = lock_db.make_lease("a")
b = lock_db.make_lease("b")
assert a.try_acquire() is True
assert b.try_acquire() is False
assert _leader(lock_db.conn) == "a"
def test_stale_lease_allows_takeover(self, lock_db):
a = lock_db.make_lease("a", lease_seconds=10)
b = lock_db.make_lease("b", lease_seconds=10)
assert a.try_acquire() is True
_age_lease(lock_db.conn, 30) # older than the 10s window
assert b.try_acquire() is True
assert _leader(lock_db.conn) == "b"
def test_same_holder_reacquire_while_fresh_returns_false(self, lock_db):
# The current holder can't re-acquire a fresh lease (the WHERE rejects
# even the holder until it goes stale) — renew is the holder's path.
a = lock_db.make_lease("a")
assert a.try_acquire() is True
assert a.try_acquire() is False
assert a.renew() is True # ...but renew keeps it
assert _leader(lock_db.conn) == "a"
def test_renew_keeps_leadership(self, lock_db):
a = lock_db.make_lease("a")
b = lock_db.make_lease("b")
assert a.try_acquire() is True
assert a.renew() is True
assert b.try_acquire() is False # still fresh after renew
def test_renew_by_non_holder_returns_false(self, lock_db):
a = lock_db.make_lease("a")
b = lock_db.make_lease("b")
assert a.try_acquire() is True
assert b.renew() is False
assert _leader(lock_db.conn) == "a" # unchanged
def test_renew_after_takeover_returns_false(self, lock_db):
# The critical safety signal: a stalled leader learns it lost the lease.
a = lock_db.make_lease("a", lease_seconds=10)
b = lock_db.make_lease("b", lease_seconds=10)
assert a.try_acquire() is True
_age_lease(lock_db.conn, 30)
assert b.try_acquire() is True # standby takes over
assert a.renew() is False # original must stop acting
def test_release_frees_immediately(self, lock_db):
a = lock_db.make_lease("a")
b = lock_db.make_lease("b")
assert a.try_acquire() is True
a.release()
assert _leader(lock_db.conn) == ""
assert b.try_acquire() is True # no need to wait out the TTL
def test_release_by_non_holder_is_noop(self, lock_db):
a = lock_db.make_lease("a")
b = lock_db.make_lease("b")
assert a.try_acquire() is True
b.release() # b doesn't hold it
assert _leader(lock_db.conn) == "a" # a still leader
assert a.renew() is True
def test_release_after_takeover_is_noop(self, lock_db):
# A stalled leader's late release must not wipe the standby that took
# over ("a late release can't wipe the new holder").
a = lock_db.make_lease("a", lease_seconds=10)
b = lock_db.make_lease("b", lease_seconds=10)
assert a.try_acquire() is True
_age_lease(lock_db.conn, 30)
assert b.try_acquire() is True # standby took over
a.release() # late release from the deposed leader
assert _leader(lock_db.conn) == "b" # new holder intact
def test_concurrent_acquire_exactly_one_wins(self, lock_db):
# N candidates, each its own connection, race try_acquire on a free lock.
# The row lock serialises the conditional UPDATE → exactly one RETURNING.
results: dict[str, bool] = {}
def run(label):
lease = lock_db.make_lease(label)
results[label] = lease.try_acquire()
labels = [f"c{i}" for i in range(5)]
threads = [threading.Thread(target=run, args=(x,)) for x in labels]
for t in threads:
t.start()
for t in threads:
t.join(timeout=10)
assert all(not t.is_alive() for t in threads)
assert sum(results.values()) == 1 # exactly one winner
winner = next(label for label, won in results.items() if won)
assert _leader(lock_db.conn) == winner
class TestSingleRowConstraint:
def test_second_row_rejected(self, lock_db):
with lock_db.conn.cursor() as cur:
with pytest.raises(psycopg2.errors.CheckViolation):
cur.execute(
"INSERT INTO pg_orchestrator_lock (id, leader, acquired_at) "
"VALUES (2, '', now())"
)
if __name__ == "__main__":
pytest.main([__file__, "-v"])