:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): keep the scheduler drain on time when the refresh is slow (#120230)
Tom Owers809e4e3c0003
diff --git a/products/warehouse_sources/backend/scheduling/runner.py b/products/warehouse_sources/backend/scheduling/runner.pyindex 836fe8c37b08..eb19864389b8 100644--- a/products/warehouse_sources/backend/scheduling/runner.py+++ b/products/warehouse_sources/backend/scheduling/runner.py@@ -1,7 +1,9 @@"""Shadow-scheduler tick loop: leader election, refresh, due scan, decisions.Every pod runs the same loop; the tick sentinel single-flights each tick across-the fleet, so followers only heartbeat. The scheduler writes decisions and+the fleet, so followers only heartbeat. The fleet-wide refresh runs in a+separate loop with its own sentinel and connection, so a slow refresh never+delays the drain of due schedules. The scheduler writes decisions andmetrics but starts no syncs — Temporal schedules stay the only thing that runswork while shadow mode is compared against them."""@@ -10,6 +12,7 @@ from __future__ import annotationsimport timeimport asyncio+import contextlibfrom collections.abc import Callablefrom datetime import UTC, datetimefrom uuid import uuid4@@ -93,6 +96,11 @@ TICKS_TOTAL = Counter(labelnames=["outcome"],)+REFRESH_FAILURES_TOTAL = Counter(+ "warehouse_pg_scheduler_refresh_failures_total",+ "Refresh loop iterations that raised",+)+@frozenclass ShadowSchedulerConfig:@@ -131,22 +139,58 @@ class ShadowScheduler:claim_limit=self._config.claim_limit,decision_retention_days=self._config.decision_retention_days,)+ refresh_task = asyncio.create_task(self._refresh_loop(shutdown))+ try:+ while not shutdown.is_set():+ started = time.monotonic()+ try:+ await self._tick(health_reporter)+ except Exception:+ # No health report on a failed tick: a pod that cannot complete+ # ticks should trip the liveness timeout and restart.+ TICKS_TOTAL.labels(outcome="error").inc()+ logger.exception("scheduler_tick_failed")+ await self._wait_for_next_cycle(shutdown, started)+ finally:+ # A refresh can run for minutes. Stopping it part way is safe because+ # the upserts are idempotent and the stale-row delete runs last.+ refresh_task.cancel()+ with contextlib.suppress(asyncio.CancelledError):+ await refresh_task+ logger.info("scheduler_stopped", owner_token=self._owner_token)++ async def _wait_for_next_cycle(self, shutdown: asyncio.Event, started: float) -> None:+ remainder = self._config.tick_interval_seconds - (time.monotonic() - started)+ if remainder > 0:+ try:+ await asyncio.wait_for(shutdown.wait(), timeout=remainder)+ except TimeoutError:+ pass++ async def _refresh_loop(self, shutdown: asyncio.Event) -> None:+ # This loop does not report health. Liveness follows the tick loop only,+ # so a pod whose ticks stop still restarts while its refresh keeps paging.while not shutdown.is_set():started = time.monotonic()try:- await self._tick(health_reporter)+ await self._refresh_if_due()except Exception:- # No health report on a failed tick: a pod that cannot complete- # ticks should trip the liveness timeout and restart.- TICKS_TOTAL.labels(outcome="error").inc()- logger.exception("scheduler_tick_failed")- remainder = self._config.tick_interval_seconds - (time.monotonic() - started)- if remainder > 0:- try:- await asyncio.wait_for(shutdown.wait(), timeout=remainder)- except TimeoutError:- pass- logger.info("scheduler_stopped", owner_token=self._owner_token)+ REFRESH_FAILURES_TOTAL.inc()+ logger.exception("scheduler_refresh_failed")+ await self._wait_for_next_cycle(shutdown, started)++ async def _refresh_if_due(self) -> bool:+ async with await psycopg.AsyncConnection.connect(self._config.database_url, autocommit=True) as conn:+ refresh_won = await JobsTable.try_acquire_sentinel_slot(+ conn,+ lane=SCHEDULER_LANE,+ group_key=REFRESH_SLOT_KEY,+ owner_token=self._owner_token,+ ttl_seconds=self._config.refresh_interval_seconds,+ )+ if refresh_won:+ await self._refresh(conn)+ return refresh_wonasync def _tick(self, health_reporter: Callable[[], None]) -> None:async with await psycopg.AsyncConnection.connect(self._config.database_url, autocommit=True) as conn:@@ -162,18 +206,6 @@ class ShadowScheduler:health_reporter()return- # Refresh before the due scan so a schema created or re-cadenced- # moments ago is judged on this tick, not the next refresh.- refresh_won = await JobsTable.try_acquire_sentinel_slot(- conn,- lane=SCHEDULER_LANE,- group_key=REFRESH_SLOT_KEY,- owner_token=self._owner_token,- ttl_seconds=self._config.refresh_interval_seconds,- )- if refresh_won:- await self._refresh(conn, health_reporter)-# Drain the backlog in claim_limit batches, one transaction each, so# a backlog larger than one claim does not make every fire late.drain_started = time.monotonic()@@ -182,21 +214,25 @@ class ShadowScheduler:decisions_inserted = 0duplicate_windows = 0missed_windows = 0+ lease_lost = Falsewhile True:outcome = await self._drain_batch(conn)+ health_reporter()+ if outcome is None:+ lease_lost = True+ breakbatches += 1due += outcome.duedecisions_inserted += outcome.decisions_insertedduplicate_windows += outcome.duplicate_windowsmissed_windows += outcome.missed_windows- health_reporter()if outcome.due < self._config.claim_limit:breakif time.monotonic() - drain_started >= self._config.drain_budget_seconds:breakDUE_PER_TICK.observe(due)- TICKS_TOTAL.labels(outcome="leader").inc()+ TICKS_TOTAL.labels(outcome="leader_lease_lost" if lease_lost else "leader").inc()logger.info("scheduler_tick",due=due,@@ -204,12 +240,19 @@ class ShadowScheduler:duplicate_windows=duplicate_windows,missed_windows=missed_windows,batches=batches,- refreshed=refresh_won,+ lease_lost=lease_lost,)- async def _drain_batch(self, conn: psycopg.AsyncConnection) -> DrainBatchOutcome:- now_epoch = int(time.time())+ async def _drain_batch(self, conn: psycopg.AsyncConnection) -> DrainBatchOutcome | None:+ """Run one claim/evaluate/advance transaction. None means the tick slot is no longer ours."""async with conn.transaction():+ # The lock holds off a new leader until this batch commits, so two pods+ # never drain at the same time, also when this batch outlives the slot.+ if not await JobsTable.lock_live_lease(+ conn, lane=SCHEDULER_LANE, group_key=TICK_SLOT_KEY, owner_token=self._owner_token+ ):+ return None+ now_epoch = int(time.time())due = await SchedulerStateTable.claim_due(conn, kind=SYNC_EXTRACT_KIND, limit=self._config.claim_limit)result = await evaluate_due(conn, due, now_epoch)inserted_records, refused = await SchedulerStateTable.insert_decisions(conn, list(result.records))@@ -263,7 +306,7 @@ class ShadowScheduler:missed_windows=inserted_missed_windows,)- async def _refresh(self, conn: psycopg.AsyncConnection, health_reporter: Callable[[], None]) -> None:+ async def _refresh(self, conn: psycopg.AsyncConnection) -> None:started = time.monotonic()# DB-clock cutoff: upserts stamp refreshed_at with now() server-side, so# the stale-row delete must compare against the same clock.@@ -287,9 +330,6 @@ class ShadowScheduler:upserts = self._build_upserts(page, now_epoch)await SchedulerStateTable.upsert_states(conn, upserts)in_scope += len(upserts)- # A full refresh can outlast the liveness timeout; a pod that keeps- # paging is healthy, and a stuck one still stops reporting.- health_reporter()deleted = await SchedulerStateTable.delete_states_not_refreshed_since(conn, SYNC_EXTRACT_KIND, refresh_start)pruned = await SchedulerStateTable.prune_decisions(diff --git a/products/warehouse_sources/backend/tests/test_scheduler_shadow.py b/products/warehouse_sources/backend/tests/test_scheduler_shadow.pyindex 81fa57803e78..cb28b661c4ca 100644--- a/products/warehouse_sources/backend/tests/test_scheduler_shadow.py+++ b/products/warehouse_sources/backend/tests/test_scheduler_shadow.py@@ -1,11 +1,11 @@import io-import mathimport timeimport uuidimport socket+import asyncioimport importlibimport urllib.request-from collections.abc import Callable+from collections.abc import Awaitable, Callablefrom datetime import (UTC,datetime,@@ -35,8 +35,11 @@ from products.warehouse_sources.backend.scheduling.shadow import (DECISION_SKIP_OUT_OF_SCOPE,DECISION_SKIP_OVERLAP,DECISION_WOULD_FIRE,+ REFRESH_SLOT_KEY,+ SCHEDULER_LANE,SYNC_EXTRACT_KIND,SYNC_EXTRACT_LANE,+ TICK_SLOT_KEY,EvaluationResult,SchemaCadence,evaluate_due,@@ -47,6 +50,7 @@ from products.warehouse_sources.backend.scheduling.shadow import (schedule_offset,window_boundary,)+from products.warehouse_sources_queue.backend.core.generic_jobs import JOB_LEASE_TABLEfrom products.warehouse_sources_queue.backend.core.scheduler_state import (SCHEDULER_DECISION_TABLE,SCHEDULER_STATE_TABLE,@@ -212,6 +216,14 @@ def _due_row(schema_id: str, team_id: int, now_epoch: int) -> DueSchedule:)+async def _wait_until(condition: Callable[[], Awaitable[bool]], timeout: float = 10.0) -> None:+ deadline = time.monotonic() + timeout+ while not await condition():+ if time.monotonic() > deadline:+ raise TimeoutError("condition not met")+ await asyncio.sleep(0.05)++def _reset_queue_jobs() -> str:db_url = get_test_database_url()with psycopg.Connection.connect(db_url, autocommit=True) as conn:@@ -371,6 +383,129 @@ class TestShadowSchedulerTick:).fetchall()return {row[0] for row in rows}+ def _set_tick_slot(self, db_url: str, assignments: str) -> None:+ with psycopg.Connection.connect(db_url, autocommit=True) as conn:+ conn.execute(+ f"UPDATE {JOB_LEASE_TABLE} SET {assignments} WHERE lane = %s AND group_key = %s",+ (SCHEDULER_LANE, TICK_SLOT_KEY),+ )++ @pytest.mark.parametrize(+ "lose_slot",+ [+ pytest.param("expires_at = now() - interval '1 second'", id="slot_expired"),+ pytest.param(+ "owner_token = 'other-pod', expires_at = now() + interval '60 seconds'", id="slot_taken_by_other_pod"+ ),+ ],+ )+ def test_leader_stops_draining_after_losing_the_tick_slot(self, team, lose_slot):+ schema_ids, db_url = self._setup_due_schemas(team, 5)+ scheduler = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url, claim_limit=2))++ # The health report runs after each committed batch, so the slot is lost between batches.+ async_to_sync(scheduler._tick)(lambda: self._set_tick_slot(db_url, lose_slot))++ assert len(self._decided_keys(db_url)) == 2+ assert len(self._overdue_keys(db_url)) == 3++ def test_new_leader_waits_for_the_batch_of_a_leader_whose_slot_expired(self, team, monkeypatch):+ schema_ids, db_url = self._setup_due_schemas(team, 4)+ old_leader = ShadowScheduler(+ ShadowSchedulerConfig(database_url=db_url, claim_limit=2, tick_interval_seconds=1.0)+ )+ new_leader = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url, claim_limit=2))+ real_evaluate_due = scheduler_runner.evaluate_due+ evaluations: list[tuple[str, str]] = []++ async def scenario() -> None:+ old_leader_in_batch = asyncio.Event()+ release_old_leader = asyncio.Event()++ async def evaluate_due(conn, due, now_epoch):+ current_task = asyncio.current_task()+ assert current_task is not None+ task_name = current_task.get_name()+ evaluations.append((task_name, "start"))+ if task_name == "old" and not old_leader_in_batch.is_set():+ old_leader_in_batch.set()+ await release_old_leader.wait()+ result = await real_evaluate_due(conn, due, now_epoch)+ evaluations.append((task_name, "end"))+ return result++ monkeypatch.setattr(scheduler_runner, "evaluate_due", evaluate_due)++ async with await psycopg.AsyncConnection.connect(db_url, autocommit=True) as probe:++ async def tick_slot_expired() -> bool:+ cursor = await probe.execute(+ f"SELECT expires_at <= now() FROM {JOB_LEASE_TABLE} WHERE lane = %s AND group_key = %s",+ (SCHEDULER_LANE, TICK_SLOT_KEY),+ )+ row = await cursor.fetchone()+ return bool(row and row[0])++ async def new_leader_blocked_or_draining() -> bool:+ if any(name == "new" for name, _ in evaluations):+ return True+ cursor = await probe.execute(+ "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database()"+ " AND wait_event_type = 'Lock' AND query LIKE %s",+ (f"%{JOB_LEASE_TABLE}%",),+ )+ row = await cursor.fetchone()+ return bool(row and row[0])++ old_tick = asyncio.create_task(old_leader._tick(lambda: None), name="old")+ await asyncio.wait_for(old_leader_in_batch.wait(), timeout=10)+ await _wait_until(tick_slot_expired)+ new_tick = asyncio.create_task(new_leader._tick(lambda: None), name="new")+ await _wait_until(new_leader_blocked_or_draining)+ release_old_leader.set()+ await asyncio.wait_for(asyncio.gather(old_tick, new_tick), timeout=10)++ async_to_sync(scenario)()++ assert evaluations[:2] == [("old", "start"), ("old", "end")]+ assert {name for name, _ in evaluations[2:]} == {"new"}+ assert self._decided_keys(db_url) == schema_ids+ assert self._overdue_keys(db_url) == set()++ @pytest.mark.parametrize("refresh_problem", ["hangs", "raises"])+ def test_refresh_problem_does_not_delay_the_drain(self, team, monkeypatch, refresh_problem):+ schema_ids, db_url = self._setup_due_schemas(team, 3)+ scheduler = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url, tick_interval_seconds=30))++ async def scenario() -> None:+ refresh_entered = asyncio.Event()++ async def refresh(conn):+ refresh_entered.set()+ if refresh_problem == "hangs":+ await asyncio.Event().wait()+ raise RuntimeError("refresh failed")++ monkeypatch.setattr(scheduler, "_refresh", refresh)++ async def all_decided() -> bool:+ async with await psycopg.AsyncConnection.connect(db_url) as conn:+ cursor = await conn.execute(f"SELECT schedule_key FROM {SCHEDULER_DECISION_TABLE}")+ return {row[0] for row in await cursor.fetchall()} == schema_ids++ shutdown = asyncio.Event()+ run = asyncio.create_task(scheduler.run(shutdown, lambda: None))+ try:+ await asyncio.wait_for(refresh_entered.wait(), timeout=10)+ await _wait_until(all_decided)+ finally:+ shutdown.set()+ await asyncio.wait_for(run, timeout=10)++ async_to_sync(scenario)()++ assert self._overdue_keys(db_url) == set()+@pytest.mark.parametrize("schema_count,claim_limit,expected_batches",[@@ -384,9 +519,6 @@ class TestShadowSchedulerTick:self, team, monkeypatch, schema_count, claim_limit, expected_batches):schema_ids, db_url = self._setup_due_schemas(team, schema_count)- monkeypatch.setattr(- scheduler_runner.JobsTable, "try_acquire_sentinel_slot", AsyncMock(side_effect=[True, False])- )due_per_tick = MagicMock()monkeypatch.setattr(scheduler_runner, "DUE_PER_TICK", due_per_tick)health_reporter = MagicMock()@@ -401,11 +533,6 @@ class TestShadowSchedulerTick:def test_drain_budget_defers_remaining_backlog_to_next_tick(self, team, monkeypatch):schema_ids, db_url = self._setup_due_schemas(team, 3)- monkeypatch.setattr(- scheduler_runner.JobsTable,- "try_acquire_sentinel_slot",- AsyncMock(side_effect=[True, False, True, False]),- )scheduler = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url, claim_limit=2, drain_budget_seconds=0))async_to_sync(scheduler._tick)(lambda: None)@@ -413,6 +540,7 @@ class TestShadowSchedulerTick:assert len(first_tick_keys) == 2assert self._overdue_keys(db_url) == schema_ids - first_tick_keys+ self._set_tick_slot(db_url, "expires_at = now() - interval '1 second'")async_to_sync(scheduler._tick)(lambda: None)assert self._decided_keys(db_url) == schema_idsassert self._overdue_keys(db_url) == set()@@ -438,11 +566,6 @@ class TestShadowSchedulerTick:await original_insert(conn, records)raise RuntimeError("decision write failed")- monkeypatch.setattr(- scheduler_runner.JobsTable,- "try_acquire_sentinel_slot",- AsyncMock(side_effect=[True, False]),- )monkeypatch.setattr(scheduler_runner,"evaluate_due",@@ -483,11 +606,6 @@ class TestShadowSchedulerTick:),)- monkeypatch.setattr(- scheduler_runner.JobsTable,- "try_acquire_sentinel_slot",- AsyncMock(side_effect=[True, False]),- )monkeypatch.setattr(scheduler_runner,"evaluate_due",@@ -513,11 +631,11 @@ class TestShadowSchedulerTick:@pytest.mark.django_db(transaction=True)class TestShadowSchedulerRefresh:- def _refresh(self, db_url: str, page_size: int, health_reporter: Callable[[], None] = lambda: None) -> None:+ def _refresh(self, db_url: str, page_size: int) -> None:async def run() -> None:scheduler = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url, refresh_page_size=page_size))async with await psycopg.AsyncConnection.connect(db_url, autocommit=True) as conn:- await scheduler._refresh(conn, health_reporter)+ await scheduler._refresh(conn)async_to_sync(run)()@@ -544,13 +662,11 @@ class TestShadowSchedulerRefresh:_create_schema(team, source, name="out_of_scope", should_sync=False)in_scope_gauge = MagicMock()monkeypatch.setattr(scheduler_runner, "SCHEMAS_IN_SCOPE", in_scope_gauge)- health_reporter = MagicMock()- self._refresh(db_url, page_size, health_reporter)+ self._refresh(db_url, page_size)assert self._state_keys(db_url) == schema_idsin_scope_gauge.set.assert_called_once_with(schema_count)- assert health_reporter.call_count >= math.ceil(schema_count / page_size)@pytest.mark.parametrize("page_size", [1, 2, 10])def test_paged_refresh_deletes_schemas_that_left_scope(self, team, page_size):@@ -567,6 +683,42 @@ class TestShadowSchedulerRefresh:assert self._state_keys(db_url) == {str(schema.id) for schema in schemas[1:4]}+ @pytest.mark.parametrize(+ "slot_owner,slot_expires_in_seconds,expected_refresh",+ [+ pytest.param(None, None, True, id="no_slot_yet"),+ pytest.param("other-pod", 300, False, id="other_pod_refreshed_this_interval"),+ pytest.param("this-pod", 300, False, id="this_pod_refreshed_this_interval"),+ pytest.param("other-pod", -1, True, id="refresh_interval_elapsed"),+ ],+ )+ def test_refresh_runs_once_per_refresh_interval(+ self, monkeypatch, slot_owner, slot_expires_in_seconds, expected_refresh+ ):+ db_url = self._reset_state()+ scheduler = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url))+ with psycopg.Connection.connect(db_url, autocommit=True) as conn:+ ensure_generic_job_tables(conn)+ truncate_generic_job_tables(conn)+ if slot_owner is not None:+ conn.execute(+ f"""+ INSERT INTO {JOB_LEASE_TABLE} (lane, group_key, owner_token, expires_at, acquired_at, updated_at)+ VALUES (%s, %s, %s, now() + make_interval(secs => %s), now(), now())+ """,+ (+ SCHEDULER_LANE,+ REFRESH_SLOT_KEY,+ scheduler._owner_token if slot_owner == "this-pod" else slot_owner,+ slot_expires_in_seconds,+ ),+ )+ refresh = AsyncMock()+ monkeypatch.setattr(scheduler, "_refresh", refresh)++ assert async_to_sync(scheduler._refresh_if_due)() is expected_refresh+ assert refresh.await_count == int(expected_refresh)+class TestSchedulerMetricsEndpoint:def test_metrics_endpoint_serves_scheduler_metrics(self):diff --git a/products/warehouse_sources_queue/backend/README.md b/products/warehouse_sources_queue/backend/README.mdindex dfdc14ab4d7c..45731c04badc 100644--- a/products/warehouse_sources_queue/backend/README.md+++ b/products/warehouse_sources_queue/backend/README.md@@ -206,6 +206,10 @@ The `run_warehouse_scheduler` command runs the tick loop.Each scheduled kind declares its queue lane and its overlap policy; `sync.extract` uses `SKIP` on lane `extract`.The scheduler decides overlap from the live jobs in `queuejob`, never from `ExternalDataJob` rows.The scheduler uses sentinel rows in `queuejoblease` (lane `scheduler`) to select one leader across the fleet.+The tick leader drains due schedules.+The fleet-wide refresh runs in a separate loop with its own sentinel row, so a slow refresh does not delay the drain.+Each drain batch locks the tick sentinel row with `JobsTable.lock_live_lease` and stops when the row is no longer this pod's.+A new leader waits for that lock, so two pods never drain at the same time.It starts no syncs.The `report_warehouse_scheduler_shadow` command compares decisions with the `ExternalDataJob` rows that Temporal schedules created.It counts an unmatched decision as `temporal_overlap_skip`, not `shadow_only`, when a job for that schema was in progress at the due time and was created at most 7 days before it.diff --git a/products/warehouse_sources_queue/backend/core/generic_jobs.py b/products/warehouse_sources_queue/backend/core/generic_jobs.pyindex a58c834ea295..8400cad13391 100644--- a/products/warehouse_sources_queue/backend/core/generic_jobs.py+++ b/products/warehouse_sources_queue/backend/core/generic_jobs.py@@ -715,6 +715,34 @@ class JobsTable:row = await cur.fetchone()return bool(row and row[0])+ @staticmethod+ async def lock_live_lease(+ conn: psycopg.AsyncConnection[Any],+ *,+ lane: str,+ group_key: str,+ owner_token: str,+ ) -> bool:+ """Confirm this owner holds a live lease and lock the lease row until the transaction ends.++ Call inside a transaction, before the work that the lease guards. A+ takeover (``try_acquire_sentinel_slot`` or a lease steal) waits on the+ row lock, so it cannot start until this transaction ends, also when the+ lease expires during the transaction. False means that the lease+ expired or has a different owner, so the caller must not do the work.+ """+ async with conn.cursor() as cur:+ await cur.execute(+ f"""+ SELECT 1 FROM {JOB_LEASE_TABLE}+ WHERE lane = %(lane)s AND group_key = %(group_key)s+ AND owner_token = %(owner)s AND expires_at > now()+ FOR UPDATE+ """,+ {"lane": lane, "group_key": group_key, "owner": owner_token},+ )+ return (await cur.fetchone()) is not None+@staticmethodasync def delete_expired_lease(conn: psycopg.AsyncConnection[Any],diff --git a/products/warehouse_sources_queue/backend/tests/test_scheduler_state.py b/products/warehouse_sources_queue/backend/tests/test_scheduler_state.pyindex 26de62ce5515..6d9a37d2bcd7 100644--- a/products/warehouse_sources_queue/backend/tests/test_scheduler_state.py+++ b/products/warehouse_sources_queue/backend/tests/test_scheduler_state.py@@ -1,3 +1,5 @@+import time+import asynciofrom datetime import UTC, datetime, timedeltafrom typing import Anyfrom uuid import uuid4@@ -218,3 +220,59 @@ class TestSchedulerState:async with conn.cursor() as cur:await cur.execute(f"UPDATE {JOB_LEASE_TABLE} SET expires_at = now() - interval '1 second'")assert await acquire(OWNER_B) is True++ @pytest.mark.parametrize(+ "lease_owner,expires_in_seconds,expected",+ [+ pytest.param(OWNER_A, 60, True, id="own_live_lease"),+ pytest.param(OWNER_A, -1, False, id="own_expired_lease"),+ pytest.param(OWNER_B, 60, False, id="other_owner_lease"),+ pytest.param(None, None, False, id="no_lease"),+ ],+ )+ @pytest.mark.asyncio+ async def test_lock_live_lease_only_holds_own_live_lease(self, conn, lease_owner, expires_in_seconds, expected):+ if lease_owner is not None:+ await JobsTable.try_acquire_sentinel_slot(+ conn, lane="scheduler", group_key="tick-slot", owner_token=lease_owner, ttl_seconds=expires_in_seconds+ )++ async with conn.transaction():+ held = await JobsTable.lock_live_lease(conn, lane="scheduler", group_key="tick-slot", owner_token=OWNER_A)++ assert held is expected++ @pytest.mark.asyncio+ async def test_lock_live_lease_blocks_takeover_until_commit(self, conn, _db_url):+ await JobsTable.try_acquire_sentinel_slot(+ conn, lane="scheduler", group_key="tick-slot", owner_token=OWNER_A, ttl_seconds=60.0+ )+ async with (+ await psycopg.AsyncConnection.connect(_db_url, autocommit=True) as other_conn,+ await psycopg.AsyncConnection.connect(_db_url, autocommit=True) as probe,+ ):+ async with conn.transaction():+ assert await JobsTable.lock_live_lease(+ conn, lane="scheduler", group_key="tick-slot", owner_token=OWNER_A+ )+ takeover = asyncio.create_task(+ JobsTable.try_acquire_sentinel_slot(+ other_conn, lane="scheduler", group_key="tick-slot", owner_token=OWNER_B, ttl_seconds=60.0+ )+ )+ deadline = time.monotonic() + 10+ # Probe from an autocommit connection, because pg_stat_activity stays+ # the same for the whole of a transaction.+ while True:+ cursor = await probe.execute(+ "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database()"+ " AND wait_event_type = 'Lock' AND query LIKE %s",+ (f"%{JOB_LEASE_TABLE}%",),+ )+ row = await cursor.fetchone()+ if row and row[0]:+ break+ assert time.monotonic() < deadline, "takeover never waited on the lease row lock"+ await asyncio.sleep(0.05)+ assert not takeover.done()+ assert await asyncio.wait_for(takeover, timeout=10) is False