Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): keep the scheduler drain on time when the refresh is slow (#120230)

Tom Owers809e4e3c0003
diff --git a/products/warehouse_sources/backend/scheduling/runner.py b/products/warehouse_sources/backend/scheduling/runner.py
index 836fe8c37b08..eb19864389b8 100644
--- a/products/warehouse_sources/backend/scheduling/runner.py
+++ b/products/warehouse_sources/backend/scheduling/runner.py
@@ -1,7 +1,9 @@
"""Shadow-scheduler tick loop: leader election, refresh, due scan, decisions.
Every pod runs the same loop; the tick sentinel single-flights each tick across
-the fleet, so followers only heartbeat. The scheduler writes decisions and
+the fleet, so followers only heartbeat. The fleet-wide refresh runs in a
+separate loop with its own sentinel and connection, so a slow refresh never
+delays the drain of due schedules. The scheduler writes decisions and
metrics but starts no syncs — Temporal schedules stay the only thing that runs
work while shadow mode is compared against them.
"""
@@ -10,6 +12,7 @@ from __future__ import annotations
import time
import asyncio
+import contextlib
from collections.abc import Callable
from datetime import UTC, datetime
from uuid import uuid4
@@ -93,6 +96,11 @@ TICKS_TOTAL = Counter(
labelnames=["outcome"],
)
+REFRESH_FAILURES_TOTAL = Counter(
+ "warehouse_pg_scheduler_refresh_failures_total",
+ "Refresh loop iterations that raised",
+)
+
@frozen
class ShadowSchedulerConfig:
@@ -131,22 +139,58 @@ class ShadowScheduler:
claim_limit=self._config.claim_limit,
decision_retention_days=self._config.decision_retention_days,
)
+ refresh_task = asyncio.create_task(self._refresh_loop(shutdown))
+ try:
+ while not shutdown.is_set():
+ started = time.monotonic()
+ try:
+ await self._tick(health_reporter)
+ except Exception:
+ # No health report on a failed tick: a pod that cannot complete
+ # ticks should trip the liveness timeout and restart.
+ TICKS_TOTAL.labels(outcome="error").inc()
+ logger.exception("scheduler_tick_failed")
+ await self._wait_for_next_cycle(shutdown, started)
+ finally:
+ # A refresh can run for minutes. Stopping it part way is safe because
+ # the upserts are idempotent and the stale-row delete runs last.
+ refresh_task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await refresh_task
+ logger.info("scheduler_stopped", owner_token=self._owner_token)
+
+ async def _wait_for_next_cycle(self, shutdown: asyncio.Event, started: float) -> None:
+ remainder = self._config.tick_interval_seconds - (time.monotonic() - started)
+ if remainder > 0:
+ try:
+ await asyncio.wait_for(shutdown.wait(), timeout=remainder)
+ except TimeoutError:
+ pass
+
+ async def _refresh_loop(self, shutdown: asyncio.Event) -> None:
+ # This loop does not report health. Liveness follows the tick loop only,
+ # so a pod whose ticks stop still restarts while its refresh keeps paging.
while not shutdown.is_set():
started = time.monotonic()
try:
- await self._tick(health_reporter)
+ await self._refresh_if_due()
except Exception:
- # No health report on a failed tick: a pod that cannot complete
- # ticks should trip the liveness timeout and restart.
- TICKS_TOTAL.labels(outcome="error").inc()
- logger.exception("scheduler_tick_failed")
- remainder = self._config.tick_interval_seconds - (time.monotonic() - started)
- if remainder > 0:
- try:
- await asyncio.wait_for(shutdown.wait(), timeout=remainder)
- except TimeoutError:
- pass
- logger.info("scheduler_stopped", owner_token=self._owner_token)
+ REFRESH_FAILURES_TOTAL.inc()
+ logger.exception("scheduler_refresh_failed")
+ await self._wait_for_next_cycle(shutdown, started)
+
+ async def _refresh_if_due(self) -> bool:
+ async with await psycopg.AsyncConnection.connect(self._config.database_url, autocommit=True) as conn:
+ refresh_won = await JobsTable.try_acquire_sentinel_slot(
+ conn,
+ lane=SCHEDULER_LANE,
+ group_key=REFRESH_SLOT_KEY,
+ owner_token=self._owner_token,
+ ttl_seconds=self._config.refresh_interval_seconds,
+ )
+ if refresh_won:
+ await self._refresh(conn)
+ return refresh_won
async def _tick(self, health_reporter: Callable[[], None]) -> None:
async with await psycopg.AsyncConnection.connect(self._config.database_url, autocommit=True) as conn:
@@ -162,18 +206,6 @@ class ShadowScheduler:
health_reporter()
return
- # Refresh before the due scan so a schema created or re-cadenced
- # moments ago is judged on this tick, not the next refresh.
- refresh_won = await JobsTable.try_acquire_sentinel_slot(
- conn,
- lane=SCHEDULER_LANE,
- group_key=REFRESH_SLOT_KEY,
- owner_token=self._owner_token,
- ttl_seconds=self._config.refresh_interval_seconds,
- )
- if refresh_won:
- await self._refresh(conn, health_reporter)
-
# Drain the backlog in claim_limit batches, one transaction each, so
# a backlog larger than one claim does not make every fire late.
drain_started = time.monotonic()
@@ -182,21 +214,25 @@ class ShadowScheduler:
decisions_inserted = 0
duplicate_windows = 0
missed_windows = 0
+ lease_lost = False
while True:
outcome = await self._drain_batch(conn)
+ health_reporter()
+ if outcome is None:
+ lease_lost = True
+ break
batches += 1
due += outcome.due
decisions_inserted += outcome.decisions_inserted
duplicate_windows += outcome.duplicate_windows
missed_windows += outcome.missed_windows
- health_reporter()
if outcome.due < self._config.claim_limit:
break
if time.monotonic() - drain_started >= self._config.drain_budget_seconds:
break
DUE_PER_TICK.observe(due)
- TICKS_TOTAL.labels(outcome="leader").inc()
+ TICKS_TOTAL.labels(outcome="leader_lease_lost" if lease_lost else "leader").inc()
logger.info(
"scheduler_tick",
due=due,
@@ -204,12 +240,19 @@ class ShadowScheduler:
duplicate_windows=duplicate_windows,
missed_windows=missed_windows,
batches=batches,
- refreshed=refresh_won,
+ lease_lost=lease_lost,
)
- async def _drain_batch(self, conn: psycopg.AsyncConnection) -> DrainBatchOutcome:
- now_epoch = int(time.time())
+ async def _drain_batch(self, conn: psycopg.AsyncConnection) -> DrainBatchOutcome | None:
+ """Run one claim/evaluate/advance transaction. None means the tick slot is no longer ours."""
async with conn.transaction():
+ # The lock holds off a new leader until this batch commits, so two pods
+ # never drain at the same time, also when this batch outlives the slot.
+ if not await JobsTable.lock_live_lease(
+ conn, lane=SCHEDULER_LANE, group_key=TICK_SLOT_KEY, owner_token=self._owner_token
+ ):
+ return None
+ now_epoch = int(time.time())
due = await SchedulerStateTable.claim_due(conn, kind=SYNC_EXTRACT_KIND, limit=self._config.claim_limit)
result = await evaluate_due(conn, due, now_epoch)
inserted_records, refused = await SchedulerStateTable.insert_decisions(conn, list(result.records))
@@ -263,7 +306,7 @@ class ShadowScheduler:
missed_windows=inserted_missed_windows,
)
- async def _refresh(self, conn: psycopg.AsyncConnection, health_reporter: Callable[[], None]) -> None:
+ async def _refresh(self, conn: psycopg.AsyncConnection) -> None:
started = time.monotonic()
# DB-clock cutoff: upserts stamp refreshed_at with now() server-side, so
# the stale-row delete must compare against the same clock.
@@ -287,9 +330,6 @@ class ShadowScheduler:
upserts = self._build_upserts(page, now_epoch)
await SchedulerStateTable.upsert_states(conn, upserts)
in_scope += len(upserts)
- # A full refresh can outlast the liveness timeout; a pod that keeps
- # paging is healthy, and a stuck one still stops reporting.
- health_reporter()
deleted = await SchedulerStateTable.delete_states_not_refreshed_since(conn, SYNC_EXTRACT_KIND, refresh_start)
pruned = await SchedulerStateTable.prune_decisions(
diff --git a/products/warehouse_sources/backend/tests/test_scheduler_shadow.py b/products/warehouse_sources/backend/tests/test_scheduler_shadow.py
index 81fa57803e78..cb28b661c4ca 100644
--- a/products/warehouse_sources/backend/tests/test_scheduler_shadow.py
+++ b/products/warehouse_sources/backend/tests/test_scheduler_shadow.py
@@ -1,11 +1,11 @@
import io
-import math
import time
import uuid
import socket
+import asyncio
import importlib
import urllib.request
-from collections.abc import Callable
+from collections.abc import Awaitable, Callable
from datetime import (
UTC,
datetime,
@@ -35,8 +35,11 @@ from products.warehouse_sources.backend.scheduling.shadow import (
DECISION_SKIP_OUT_OF_SCOPE,
DECISION_SKIP_OVERLAP,
DECISION_WOULD_FIRE,
+ REFRESH_SLOT_KEY,
+ SCHEDULER_LANE,
SYNC_EXTRACT_KIND,
SYNC_EXTRACT_LANE,
+ TICK_SLOT_KEY,
EvaluationResult,
SchemaCadence,
evaluate_due,
@@ -47,6 +50,7 @@ from products.warehouse_sources.backend.scheduling.shadow import (
schedule_offset,
window_boundary,
)
+from products.warehouse_sources_queue.backend.core.generic_jobs import JOB_LEASE_TABLE
from products.warehouse_sources_queue.backend.core.scheduler_state import (
SCHEDULER_DECISION_TABLE,
SCHEDULER_STATE_TABLE,
@@ -212,6 +216,14 @@ def _due_row(schema_id: str, team_id: int, now_epoch: int) -> DueSchedule:
)
+async def _wait_until(condition: Callable[[], Awaitable[bool]], timeout: float = 10.0) -> None:
+ deadline = time.monotonic() + timeout
+ while not await condition():
+ if time.monotonic() > deadline:
+ raise TimeoutError("condition not met")
+ await asyncio.sleep(0.05)
+
+
def _reset_queue_jobs() -> str:
db_url = get_test_database_url()
with psycopg.Connection.connect(db_url, autocommit=True) as conn:
@@ -371,6 +383,129 @@ class TestShadowSchedulerTick:
).fetchall()
return {row[0] for row in rows}
+ def _set_tick_slot(self, db_url: str, assignments: str) -> None:
+ with psycopg.Connection.connect(db_url, autocommit=True) as conn:
+ conn.execute(
+ f"UPDATE {JOB_LEASE_TABLE} SET {assignments} WHERE lane = %s AND group_key = %s",
+ (SCHEDULER_LANE, TICK_SLOT_KEY),
+ )
+
+ @pytest.mark.parametrize(
+ "lose_slot",
+ [
+ pytest.param("expires_at = now() - interval '1 second'", id="slot_expired"),
+ pytest.param(
+ "owner_token = 'other-pod', expires_at = now() + interval '60 seconds'", id="slot_taken_by_other_pod"
+ ),
+ ],
+ )
+ def test_leader_stops_draining_after_losing_the_tick_slot(self, team, lose_slot):
+ schema_ids, db_url = self._setup_due_schemas(team, 5)
+ scheduler = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url, claim_limit=2))
+
+ # The health report runs after each committed batch, so the slot is lost between batches.
+ async_to_sync(scheduler._tick)(lambda: self._set_tick_slot(db_url, lose_slot))
+
+ assert len(self._decided_keys(db_url)) == 2
+ assert len(self._overdue_keys(db_url)) == 3
+
+ def test_new_leader_waits_for_the_batch_of_a_leader_whose_slot_expired(self, team, monkeypatch):
+ schema_ids, db_url = self._setup_due_schemas(team, 4)
+ old_leader = ShadowScheduler(
+ ShadowSchedulerConfig(database_url=db_url, claim_limit=2, tick_interval_seconds=1.0)
+ )
+ new_leader = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url, claim_limit=2))
+ real_evaluate_due = scheduler_runner.evaluate_due
+ evaluations: list[tuple[str, str]] = []
+
+ async def scenario() -> None:
+ old_leader_in_batch = asyncio.Event()
+ release_old_leader = asyncio.Event()
+
+ async def evaluate_due(conn, due, now_epoch):
+ current_task = asyncio.current_task()
+ assert current_task is not None
+ task_name = current_task.get_name()
+ evaluations.append((task_name, "start"))
+ if task_name == "old" and not old_leader_in_batch.is_set():
+ old_leader_in_batch.set()
+ await release_old_leader.wait()
+ result = await real_evaluate_due(conn, due, now_epoch)
+ evaluations.append((task_name, "end"))
+ return result
+
+ monkeypatch.setattr(scheduler_runner, "evaluate_due", evaluate_due)
+
+ async with await psycopg.AsyncConnection.connect(db_url, autocommit=True) as probe:
+
+ async def tick_slot_expired() -> bool:
+ cursor = await probe.execute(
+ f"SELECT expires_at <= now() FROM {JOB_LEASE_TABLE} WHERE lane = %s AND group_key = %s",
+ (SCHEDULER_LANE, TICK_SLOT_KEY),
+ )
+ row = await cursor.fetchone()
+ return bool(row and row[0])
+
+ async def new_leader_blocked_or_draining() -> bool:
+ if any(name == "new" for name, _ in evaluations):
+ return True
+ cursor = await probe.execute(
+ "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database()"
+ " AND wait_event_type = 'Lock' AND query LIKE %s",
+ (f"%{JOB_LEASE_TABLE}%",),
+ )
+ row = await cursor.fetchone()
+ return bool(row and row[0])
+
+ old_tick = asyncio.create_task(old_leader._tick(lambda: None), name="old")
+ await asyncio.wait_for(old_leader_in_batch.wait(), timeout=10)
+ await _wait_until(tick_slot_expired)
+ new_tick = asyncio.create_task(new_leader._tick(lambda: None), name="new")
+ await _wait_until(new_leader_blocked_or_draining)
+ release_old_leader.set()
+ await asyncio.wait_for(asyncio.gather(old_tick, new_tick), timeout=10)
+
+ async_to_sync(scenario)()
+
+ assert evaluations[:2] == [("old", "start"), ("old", "end")]
+ assert {name for name, _ in evaluations[2:]} == {"new"}
+ assert self._decided_keys(db_url) == schema_ids
+ assert self._overdue_keys(db_url) == set()
+
+ @pytest.mark.parametrize("refresh_problem", ["hangs", "raises"])
+ def test_refresh_problem_does_not_delay_the_drain(self, team, monkeypatch, refresh_problem):
+ schema_ids, db_url = self._setup_due_schemas(team, 3)
+ scheduler = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url, tick_interval_seconds=30))
+
+ async def scenario() -> None:
+ refresh_entered = asyncio.Event()
+
+ async def refresh(conn):
+ refresh_entered.set()
+ if refresh_problem == "hangs":
+ await asyncio.Event().wait()
+ raise RuntimeError("refresh failed")
+
+ monkeypatch.setattr(scheduler, "_refresh", refresh)
+
+ async def all_decided() -> bool:
+ async with await psycopg.AsyncConnection.connect(db_url) as conn:
+ cursor = await conn.execute(f"SELECT schedule_key FROM {SCHEDULER_DECISION_TABLE}")
+ return {row[0] for row in await cursor.fetchall()} == schema_ids
+
+ shutdown = asyncio.Event()
+ run = asyncio.create_task(scheduler.run(shutdown, lambda: None))
+ try:
+ await asyncio.wait_for(refresh_entered.wait(), timeout=10)
+ await _wait_until(all_decided)
+ finally:
+ shutdown.set()
+ await asyncio.wait_for(run, timeout=10)
+
+ async_to_sync(scenario)()
+
+ assert self._overdue_keys(db_url) == set()
+
@pytest.mark.parametrize(
"schema_count,claim_limit,expected_batches",
[
@@ -384,9 +519,6 @@ class TestShadowSchedulerTick:
self, team, monkeypatch, schema_count, claim_limit, expected_batches
):
schema_ids, db_url = self._setup_due_schemas(team, schema_count)
- monkeypatch.setattr(
- scheduler_runner.JobsTable, "try_acquire_sentinel_slot", AsyncMock(side_effect=[True, False])
- )
due_per_tick = MagicMock()
monkeypatch.setattr(scheduler_runner, "DUE_PER_TICK", due_per_tick)
health_reporter = MagicMock()
@@ -401,11 +533,6 @@ class TestShadowSchedulerTick:
def test_drain_budget_defers_remaining_backlog_to_next_tick(self, team, monkeypatch):
schema_ids, db_url = self._setup_due_schemas(team, 3)
- monkeypatch.setattr(
- scheduler_runner.JobsTable,
- "try_acquire_sentinel_slot",
- AsyncMock(side_effect=[True, False, True, False]),
- )
scheduler = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url, claim_limit=2, drain_budget_seconds=0))
async_to_sync(scheduler._tick)(lambda: None)
@@ -413,6 +540,7 @@ class TestShadowSchedulerTick:
assert len(first_tick_keys) == 2
assert self._overdue_keys(db_url) == schema_ids - first_tick_keys
+ self._set_tick_slot(db_url, "expires_at = now() - interval '1 second'")
async_to_sync(scheduler._tick)(lambda: None)
assert self._decided_keys(db_url) == schema_ids
assert self._overdue_keys(db_url) == set()
@@ -438,11 +566,6 @@ class TestShadowSchedulerTick:
await original_insert(conn, records)
raise RuntimeError("decision write failed")
- monkeypatch.setattr(
- scheduler_runner.JobsTable,
- "try_acquire_sentinel_slot",
- AsyncMock(side_effect=[True, False]),
- )
monkeypatch.setattr(
scheduler_runner,
"evaluate_due",
@@ -483,11 +606,6 @@ class TestShadowSchedulerTick:
),
)
- monkeypatch.setattr(
- scheduler_runner.JobsTable,
- "try_acquire_sentinel_slot",
- AsyncMock(side_effect=[True, False]),
- )
monkeypatch.setattr(
scheduler_runner,
"evaluate_due",
@@ -513,11 +631,11 @@ class TestShadowSchedulerTick:
@pytest.mark.django_db(transaction=True)
class TestShadowSchedulerRefresh:
- def _refresh(self, db_url: str, page_size: int, health_reporter: Callable[[], None] = lambda: None) -> None:
+ def _refresh(self, db_url: str, page_size: int) -> None:
async def run() -> None:
scheduler = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url, refresh_page_size=page_size))
async with await psycopg.AsyncConnection.connect(db_url, autocommit=True) as conn:
- await scheduler._refresh(conn, health_reporter)
+ await scheduler._refresh(conn)
async_to_sync(run)()
@@ -544,13 +662,11 @@ class TestShadowSchedulerRefresh:
_create_schema(team, source, name="out_of_scope", should_sync=False)
in_scope_gauge = MagicMock()
monkeypatch.setattr(scheduler_runner, "SCHEMAS_IN_SCOPE", in_scope_gauge)
- health_reporter = MagicMock()
- self._refresh(db_url, page_size, health_reporter)
+ self._refresh(db_url, page_size)
assert self._state_keys(db_url) == schema_ids
in_scope_gauge.set.assert_called_once_with(schema_count)
- assert health_reporter.call_count >= math.ceil(schema_count / page_size)
@pytest.mark.parametrize("page_size", [1, 2, 10])
def test_paged_refresh_deletes_schemas_that_left_scope(self, team, page_size):
@@ -567,6 +683,42 @@ class TestShadowSchedulerRefresh:
assert self._state_keys(db_url) == {str(schema.id) for schema in schemas[1:4]}
+ @pytest.mark.parametrize(
+ "slot_owner,slot_expires_in_seconds,expected_refresh",
+ [
+ pytest.param(None, None, True, id="no_slot_yet"),
+ pytest.param("other-pod", 300, False, id="other_pod_refreshed_this_interval"),
+ pytest.param("this-pod", 300, False, id="this_pod_refreshed_this_interval"),
+ pytest.param("other-pod", -1, True, id="refresh_interval_elapsed"),
+ ],
+ )
+ def test_refresh_runs_once_per_refresh_interval(
+ self, monkeypatch, slot_owner, slot_expires_in_seconds, expected_refresh
+ ):
+ db_url = self._reset_state()
+ scheduler = ShadowScheduler(ShadowSchedulerConfig(database_url=db_url))
+ with psycopg.Connection.connect(db_url, autocommit=True) as conn:
+ ensure_generic_job_tables(conn)
+ truncate_generic_job_tables(conn)
+ if slot_owner is not None:
+ conn.execute(
+ f"""
+ INSERT INTO {JOB_LEASE_TABLE} (lane, group_key, owner_token, expires_at, acquired_at, updated_at)
+ VALUES (%s, %s, %s, now() + make_interval(secs => %s), now(), now())
+ """,
+ (
+ SCHEDULER_LANE,
+ REFRESH_SLOT_KEY,
+ scheduler._owner_token if slot_owner == "this-pod" else slot_owner,
+ slot_expires_in_seconds,
+ ),
+ )
+ refresh = AsyncMock()
+ monkeypatch.setattr(scheduler, "_refresh", refresh)
+
+ assert async_to_sync(scheduler._refresh_if_due)() is expected_refresh
+ assert refresh.await_count == int(expected_refresh)
+
class TestSchedulerMetricsEndpoint:
def test_metrics_endpoint_serves_scheduler_metrics(self):
diff --git a/products/warehouse_sources_queue/backend/README.md b/products/warehouse_sources_queue/backend/README.md
index dfdc14ab4d7c..45731c04badc 100644
--- a/products/warehouse_sources_queue/backend/README.md
+++ b/products/warehouse_sources_queue/backend/README.md
@@ -206,6 +206,10 @@ The `run_warehouse_scheduler` command runs the tick loop.
Each scheduled kind declares its queue lane and its overlap policy; `sync.extract` uses `SKIP` on lane `extract`.
The scheduler decides overlap from the live jobs in `queuejob`, never from `ExternalDataJob` rows.
The scheduler uses sentinel rows in `queuejoblease` (lane `scheduler`) to select one leader across the fleet.
+The tick leader drains due schedules.
+The fleet-wide refresh runs in a separate loop with its own sentinel row, so a slow refresh does not delay the drain.
+Each drain batch locks the tick sentinel row with `JobsTable.lock_live_lease` and stops when the row is no longer this pod's.
+A new leader waits for that lock, so two pods never drain at the same time.
It starts no syncs.
The `report_warehouse_scheduler_shadow` command compares decisions with the `ExternalDataJob` rows that Temporal schedules created.
It counts an unmatched decision as `temporal_overlap_skip`, not `shadow_only`, when a job for that schema was in progress at the due time and was created at most 7 days before it.
diff --git a/products/warehouse_sources_queue/backend/core/generic_jobs.py b/products/warehouse_sources_queue/backend/core/generic_jobs.py
index a58c834ea295..8400cad13391 100644
--- a/products/warehouse_sources_queue/backend/core/generic_jobs.py
+++ b/products/warehouse_sources_queue/backend/core/generic_jobs.py
@@ -715,6 +715,34 @@ class JobsTable:
row = await cur.fetchone()
return bool(row and row[0])
+ @staticmethod
+ async def lock_live_lease(
+ conn: psycopg.AsyncConnection[Any],
+ *,
+ lane: str,
+ group_key: str,
+ owner_token: str,
+ ) -> bool:
+ """Confirm this owner holds a live lease and lock the lease row until the transaction ends.
+
+ Call inside a transaction, before the work that the lease guards. A
+ takeover (``try_acquire_sentinel_slot`` or a lease steal) waits on the
+ row lock, so it cannot start until this transaction ends, also when the
+ lease expires during the transaction. False means that the lease
+ expired or has a different owner, so the caller must not do the work.
+ """
+ async with conn.cursor() as cur:
+ await cur.execute(
+ f"""
+ SELECT 1 FROM {JOB_LEASE_TABLE}
+ WHERE lane = %(lane)s AND group_key = %(group_key)s
+ AND owner_token = %(owner)s AND expires_at > now()
+ FOR UPDATE
+ """,
+ {"lane": lane, "group_key": group_key, "owner": owner_token},
+ )
+ return (await cur.fetchone()) is not None
+
@staticmethod
async def delete_expired_lease(
conn: psycopg.AsyncConnection[Any],
diff --git a/products/warehouse_sources_queue/backend/tests/test_scheduler_state.py b/products/warehouse_sources_queue/backend/tests/test_scheduler_state.py
index 26de62ce5515..6d9a37d2bcd7 100644
--- a/products/warehouse_sources_queue/backend/tests/test_scheduler_state.py
+++ b/products/warehouse_sources_queue/backend/tests/test_scheduler_state.py
@@ -1,3 +1,5 @@
+import time
+import asyncio
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
@@ -218,3 +220,59 @@ class TestSchedulerState:
async with conn.cursor() as cur:
await cur.execute(f"UPDATE {JOB_LEASE_TABLE} SET expires_at = now() - interval '1 second'")
assert await acquire(OWNER_B) is True
+
+ @pytest.mark.parametrize(
+ "lease_owner,expires_in_seconds,expected",
+ [
+ pytest.param(OWNER_A, 60, True, id="own_live_lease"),
+ pytest.param(OWNER_A, -1, False, id="own_expired_lease"),
+ pytest.param(OWNER_B, 60, False, id="other_owner_lease"),
+ pytest.param(None, None, False, id="no_lease"),
+ ],
+ )
+ @pytest.mark.asyncio
+ async def test_lock_live_lease_only_holds_own_live_lease(self, conn, lease_owner, expires_in_seconds, expected):
+ if lease_owner is not None:
+ await JobsTable.try_acquire_sentinel_slot(
+ conn, lane="scheduler", group_key="tick-slot", owner_token=lease_owner, ttl_seconds=expires_in_seconds
+ )
+
+ async with conn.transaction():
+ held = await JobsTable.lock_live_lease(conn, lane="scheduler", group_key="tick-slot", owner_token=OWNER_A)
+
+ assert held is expected
+
+ @pytest.mark.asyncio
+ async def test_lock_live_lease_blocks_takeover_until_commit(self, conn, _db_url):
+ await JobsTable.try_acquire_sentinel_slot(
+ conn, lane="scheduler", group_key="tick-slot", owner_token=OWNER_A, ttl_seconds=60.0
+ )
+ async with (
+ await psycopg.AsyncConnection.connect(_db_url, autocommit=True) as other_conn,
+ await psycopg.AsyncConnection.connect(_db_url, autocommit=True) as probe,
+ ):
+ async with conn.transaction():
+ assert await JobsTable.lock_live_lease(
+ conn, lane="scheduler", group_key="tick-slot", owner_token=OWNER_A
+ )
+ takeover = asyncio.create_task(
+ JobsTable.try_acquire_sentinel_slot(
+ other_conn, lane="scheduler", group_key="tick-slot", owner_token=OWNER_B, ttl_seconds=60.0
+ )
+ )
+ deadline = time.monotonic() + 10
+ # Probe from an autocommit connection, because pg_stat_activity stays
+ # the same for the whole of a transaction.
+ while True:
+ cursor = await probe.execute(
+ "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database()"
+ " AND wait_event_type = 'Lock' AND query LIKE %s",
+ (f"%{JOB_LEASE_TABLE}%",),
+ )
+ row = await cursor.fetchone()
+ if row and row[0]:
+ break
+ assert time.monotonic() < deadline, "takeover never waited on the lease row lock"
+ await asyncio.sleep(0.05)
+ assert not takeover.done()
+ assert await asyncio.wait_for(takeover, timeout=10) is False