Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): fail runs whose batches age out of the claim window (#119691)

Tom Owers1aee83ac6f8b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py
index ef9989b6b3f6..b361680ca6c5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py
@@ -17,6 +17,11 @@ IDEMPOTENCY_HIT_TOTAL = Counter(
"Total idempotency cache hits (batch already processed)",
)
+INCOMPLETE_RUN_COMPLETIONS_REFUSED_TOTAL = Counter(
+ "warehouse_load_incomplete_run_completions_refused_total",
+ "Final batches that did not complete their job because an earlier batch of the run did not load",
+)
+
APPEND_ROLLBACK_TOTAL = Counter(
"warehouse_load_append_rollback_total",
"Append rollback attempts by outcome",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
index 07dce69f7c86..894839edfcde 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
@@ -96,6 +96,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
DELTA_ROWS_WRITTEN_TOTAL,
DELTA_WRITE_DURATION_SECONDS,
IDEMPOTENCY_HIT_TOTAL,
+ INCOMPLETE_RUN_COMPLETIONS_REFUSED_TOTAL,
PARQUET_READ_DURATION_SECONDS,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.table_handles import (
@@ -121,6 +122,7 @@ from products.warehouse_sources.backend.temporal.data_imports.util import prepar
from products.warehouse_sources.backend.temporal.data_imports.workload_report import report_phase, workload_reporting
from products.warehouse_sources.backend.types import ExternalDataSourceType
from products.warehouse_sources_queue.backend.core.batch_consumer import CoalescingDeclined, OwnershipLostError
+from products.warehouse_sources_queue.backend.core.jobs_db import BatchQueue
logger = structlog.get_logger(__name__)
@@ -132,6 +134,20 @@ FILE_COUNT_LOG_SAMPLE_EVERY = 100
REBUILD_COMPACT_CHECK_EVERY = 10
REBUILD_COMPACTION_FLAG = "warehouse-rebuild-run-compaction"
+# Shown to the customer verbatim as latest_error. The consumer also matches it to fail the run
+# without retries, because the earlier batches cannot load on a later attempt of this run.
+INCOMPLETE_RUN_ERROR = (
+ "Some batches of this sync run did not load, so the sync did not complete. "
+ "The next scheduled sync retries automatically. No action is needed."
+)
+
+
+class IncompleteRunError(Exception):
+ """A final batch arrived while earlier batches of its run had not loaded."""
+
+ def __init__(self) -> None:
+ super().__init__(INCOMPLETE_RUN_ERROR)
+
def _get_write_type(sync_type: SyncTypeLiteral) -> Literal["incremental", "full_refresh", "append"]:
"""Convert sync type to write type for DeltaTableRef."""
@@ -546,7 +562,45 @@ def _release_pipeline_lock_for_job(export_signal: ExportSignalMessage) -> None:
capture_exception(e)
+def _refuse_incomplete_run(export_signal: ExportSignalMessage) -> None:
+ """Raise when an earlier batch of the run did not load, before the job completes.
+
+ The claim gates do not see a batch that aged out of the claim window, so the later batches of
+ its run can still load. Completing such a run would promote the staged cursor past rows that
+ never loaded, and the next sync would never read them again. A failed job keeps the committed
+ cursor, so the next sync extracts those rows again.
+
+ ``batch_index`` is the first batch this write loaded for the run, so every batch of the run
+ below it loaded in an earlier write.
+ """
+ unfinished = BatchQueue.get_unfinished_earlier_batches_sync(
+ settings.WAREHOUSE_SOURCES_DATABASE_URL,
+ team_id=export_signal.team_id,
+ schema_id=export_signal.schema_id,
+ run_uuid=export_signal.run_uuid,
+ before_batch_index=export_signal.batch_index,
+ )
+ if unfinished.count == 0:
+ return
+
+ INCOMPLETE_RUN_COMPLETIONS_REFUSED_TOTAL.inc()
+ logger.error(
+ "run_completion_refused_unfinished_batches",
+ team_id=export_signal.team_id,
+ external_data_schema_id=export_signal.schema_id,
+ external_data_job_id=export_signal.job_id,
+ run_uuid=export_signal.run_uuid,
+ before_batch_index=export_signal.batch_index,
+ unfinished_batches=unfinished.count,
+ first_unfinished_batch_index=unfinished.first_batch_index,
+ unfinished_states=list(unfinished.states),
+ )
+ raise IncompleteRunError()
+
+
def _mark_job_completed(export_signal: ExportSignalMessage) -> None:
+ _refuse_incomplete_run(export_signal)
+
# Reconnect stale connections before the transaction; close_old_connections must never
# run inside an atomic block since it can drop the connection mid-transaction.
close_old_connections()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
index e2bb8d598a33..258d0f773a92 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
@@ -37,6 +37,8 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.pos
post_load_phase,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor import (
+ INCOMPLETE_RUN_ERROR,
+ IncompleteRunError,
_apply_partitioning,
_enrich_cdc_rows,
_file_count_after_write,
@@ -58,6 +60,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
from products.warehouse_sources.backend.temporal.data_imports.pipelines.test_mocks import mock_delta_table
from products.warehouse_sources.backend.types import ExternalDataSourceType
from products.warehouse_sources_queue.backend.core.batch_consumer import CoalescingDeclined
+from products.warehouse_sources_queue.backend.core.jobs_db import UnfinishedEarlierBatches
+
+_COMPLETE_RUN = UnfinishedEarlierBatches(count=0, first_batch_index=None, states=())
@pytest.fixture(autouse=True)
@@ -71,6 +76,16 @@ def _no_close_old_connections():
yield
[email protected](autouse=True)
+def _unfinished_earlier_batches() -> Iterator[MagicMock]:
+ # The completion check reads the queue DB, which these unit tests do not have.
+ with patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.BatchQueue.get_unfinished_earlier_batches_sync",
+ return_value=_COMPLETE_RUN,
+ ) as check:
+ yield check
+
+
@pytest.fixture(autouse=True)
def _table_handles() -> Iterator[GroupTableHandles]:
# The loader's registry is process-wide, so a handle that one test left would reach the next.
@@ -524,6 +539,50 @@ class TestMarkJobCompleted:
mock_finish_row_tracking.assert_not_awaited()
mock_release.assert_not_called()
+ @parameterized.expand(
+ [
+ (
+ "aged_out_batch_never_loaded",
+ UnfinishedEarlierBatches(count=1, first_batch_index=2, states=("pending",)),
+ ),
+ ("earlier_batch_failed", UnfinishedEarlierBatches(count=1, first_batch_index=0, states=("failed",))),
+ (
+ "several_batches_left",
+ UnfinishedEarlierBatches(count=3, first_batch_index=1, states=("pending", "waiting_retry")),
+ ),
+ ]
+ )
+ @patch(f"{_PROCESSOR}.release_v3_pipeline_lock")
+ @patch(f"{_PROCESSOR}.ExternalDataSchema.objects")
+ @patch(f"{_PROCESSOR}.update_external_job_status")
+ def test_a_run_with_an_unfinished_earlier_batch_does_not_complete(
+ self,
+ _case: str,
+ unfinished: UnfinishedEarlierBatches,
+ mock_update: MagicMock,
+ mock_schema_objects: MagicMock,
+ mock_release: MagicMock,
+ ) -> None:
+ signal = self._make_signal()
+ signal.batch_index = 5
+
+ with (
+ patch(f"{_PROCESSOR}.BatchQueue.get_unfinished_earlier_batches_sync", return_value=unfinished) as check,
+ pytest.raises(IncompleteRunError, match=INCOMPLETE_RUN_ERROR),
+ ):
+ _mark_job_completed(signal)
+
+ assert check.call_args.kwargs == {
+ "team_id": 1,
+ "schema_id": "schema-1",
+ "run_uuid": "run-abc-a1",
+ "before_batch_index": 5,
+ }
+ mock_update.assert_not_called()
+ mock_schema_objects.get.return_value.promote_staged_incremental_values.assert_not_called()
+ # The consumer fails the run on this error, and its fail path releases the lock.
+ mock_release.assert_not_called()
+
class TestRedeliveredFinalBatchPostLoad:
@parameterized.expand([("companion", "scd2_append"), ("consolidated", "incremental_merge"), ("snapshot", None)])
@@ -824,6 +883,9 @@ class TestProcessMessages:
# Without the flag the post-import workflow opens the table again to measure a size that
# post-load already recorded.
mock_trigger.assert_called_once_with(ANY, True)
+ # The completion check counts the batches below this index as earlier writes. The tail index
+ # would count the set's own members, which are still executing, and fail every coalesced run.
+ assert mock_mark_completed.call_args.args[0].batch_index == 0
@patch(f"{_PROCESSOR}.posthoganalytics")
@patch(f"{_PROCESSOR}.read_parquet")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
index 0a4d5e78b729..3b263cd69f07 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py
@@ -45,6 +45,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.destinations_load.errors import (
DESTINATION_CONFIGURATION_ERROR_MARKER,
)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor import (
+ INCOMPLETE_RUN_ERROR,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.table_handles import (
release_group_table_handle,
)
@@ -74,6 +77,7 @@ from products.warehouse_sources_queue.backend.core.jobs_db import (
FRESHNESS_WINDOW_SECONDS,
GAUGE_STATEMENT_TIMEOUT_MS,
TAKEOVER_STALE_THRESHOLD_SECONDS,
+ AgedOutRunRef,
BatchQueue,
ClaimCursor,
ExecutingAttempt,
@@ -94,6 +98,7 @@ from products.warehouse_sources_queue.backend.core.metrics import (
DRAINED_AFTER_FAILURE_TOTAL,
OLDEST_UNCLAIMED_BATCH_SECONDS,
ORPHANED_BATCHES_DRAINED_TOTAL,
+ RUNS_FAILED_AGED_OUT_TOTAL,
RUNS_RECONCILED_TOTAL,
RUNS_TERMINALIZED_STALE_TOTAL,
SERIALIZED_BATCHES,
@@ -134,6 +139,13 @@ STRANDED_RUN_ERROR = (
"The next scheduled sync retries automatically. No action is needed."
)
+# Stamped on a run with a batch that aged out of the claim window before it loaded. Shown to the
+# customer verbatim as latest_error, like STRANDED_RUN_ERROR.
+AGED_OUT_RUN_ERROR = (
+ "Part of this sync run waited too long to load, so the sync did not complete. "
+ "The next scheduled sync retries automatically. No action is needed."
+)
+
# Stamped on batches retired by the orphan drain. Their run already failed and the customer has
# already seen that failure on the job, so this text only has to explain the batch rows themselves.
ORPHANED_BATCH_ERROR = "left over from a run that had already failed (orphan drain)"
@@ -185,6 +197,8 @@ NON_RETRYABLE_ERROR_PATTERNS: tuple[str, ...] = (
# a destination's own settings refuse the connection (bad credentials, unknown database,
# unroutable host); the next scheduled run tries again after the customer fixes them
DESTINATION_CONFIGURATION_ERROR_MARKER,
+ # an earlier batch of the run did not load, and a retry of the final batch cannot load it
+ INCOMPLETE_RUN_ERROR,
)
# Subset of the non-retryable errors that are expected upstream/customer conditions rather than
@@ -567,6 +581,20 @@ class DeltaBatchConsumerAdapter:
logger.exception("orphaned_batch_drain_failed")
capture_exception(e)
+ # Before the stranded pass, so that a run with an aged-out batch gets this reason. Isolated so
+ # its failure can't take the sweep down.
+ try:
+ await self._fail_aged_out_runs(conn, limit=limit)
+ except psycopg.OperationalError as e:
+ if _is_transient_queue_connection_drop(e, conn):
+ logger.warning("aged_out_run_sweep_closed_connection", error=str(e))
+ else:
+ logger.exception("aged_out_run_sweep_failed")
+ capture_exception(e)
+ except Exception as e:
+ logger.exception("aged_out_run_sweep_failed")
+ capture_exception(e)
+
# Runs the loader abandoned leave no 'failed' batch for get_failed_runs to key on, so sweep
# them on the same cadence and connection. Isolated so its failure can't take the sweep down.
try:
@@ -701,6 +729,91 @@ class DeltaBatchConsumerAdapter:
)
capture_exception(e)
+ async def _fail_aged_out_runs(self, conn: psycopg.AsyncConnection[Any], *, limit: int) -> None:
+ """Fail runs with a batch that aged out of the claim window before it loaded.
+
+ The batch never loads, but the run gates do not see it, so the later batches of the run stay
+ claimable. Without this pass the final batch could complete the job and promote the cursor
+ past the rows that never loaded. The stranded-run pass does not catch it, because the group
+ makes progress on those later batches.
+
+ Batches are failed FIRST, for the same reason as in ``_reconcile_stale_stranded_runs``: the
+ later batches are still claimable, and a crash after this write leaves a failed batch for
+ ``reconcile_failed_runs`` to finalize the job from.
+ """
+ with observe_queue_query("get_aged_out_runs"):
+ refs = await BatchQueue.get_aged_out_runs(conn, limit=limit)
+ for ref in refs:
+ try:
+ failed_batches = await BatchQueue.fail_run(
+ conn,
+ run_uuid=ref.run_uuid,
+ team_id=ref.team_id,
+ schema_id=ref.schema_id,
+ reason=AGED_OUT_RUN_ERROR,
+ )
+ except Exception as e:
+ if _is_transient_queue_connection_drop(e, conn):
+ raise
+ logger.exception("aged_out_run_batch_sweep_failed", run_uuid=ref.run_uuid)
+ capture_exception(e)
+ continue
+
+ job_reconciled = await self._mark_aged_out_job_failed(ref)
+ RUNS_FAILED_AGED_OUT_TOTAL.inc()
+ logger.warning(
+ "aged_out_run_failed",
+ job_id=ref.job_id,
+ run_uuid=ref.run_uuid,
+ team_id=ref.team_id,
+ external_data_schema_id=ref.schema_id,
+ first_aged_out_batch_index=ref.first_aged_out_batch_index,
+ aged_out_batches=ref.aged_out_batches,
+ failed_batches=failed_batches,
+ job_reconciled=job_reconciled,
+ )
+
+ if ref.workflow_run_id:
+ try:
+ await sync_to_async(release_v3_pipeline_lock)(
+ team_id=ref.team_id,
+ schema_id=ref.schema_id,
+ token=ref.workflow_run_id,
+ )
+ except Exception as e:
+ logger.error(
+ "failed_to_release_v3_pipeline_lock",
+ job_id=ref.job_id,
+ schema_id=ref.schema_id,
+ exc_info=True,
+ )
+ capture_exception(e)
+
+ async def _mark_aged_out_job_failed(self, ref: AgedOutRunRef) -> bool:
+ """Write the run's failure to the app DB. Returns whether this call moved the job to Failed.
+
+ A miss is safe: the run's batches are already failed, so ``reconcile_failed_runs`` finishes
+ the job on a later cycle.
+ """
+ try:
+ return await sync_to_async(mark_job_failed_if_not_terminal)(
+ job_id=ref.job_id,
+ team_id=ref.team_id,
+ error=AGED_OUT_RUN_ERROR,
+ )
+ except Exception as e:
+ if is_transient_internal_db_error(e):
+ logger.warning(
+ "aged_out_run_job_status_update_app_db_not_ready",
+ job_id=ref.job_id,
+ run_uuid=ref.run_uuid,
+ error=str(e),
+ )
+ else:
+ logger.exception("aged_out_run_job_status_update_failed", job_id=ref.job_id, run_uuid=ref.run_uuid)
+ capture_exception(e)
+ return False
+
async def _reconcile_stale_stranded_runs(
self,
conn: psycopg.AsyncConnection[Any],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
index e7d2a9897d75..4b7b90eac27f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py
@@ -2518,10 +2518,18 @@ class TestReconcileFailedRuns:
@pytest.fixture(autouse=True)
def _hold_sweep_slot(self):
"""Grant the fleet-wide sweep slot by default — these tests exercise the sweep body, not the slot."""
- with patch(
- "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.try_acquire_reconcile_sweep_slot",
- new_callable=AsyncMock,
- return_value=True,
+ with (
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.try_acquire_reconcile_sweep_slot",
+ new_callable=AsyncMock,
+ return_value=True,
+ ),
+ # The aged-out pass has its own tests against a real queue in test_jobs_db.py.
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.get_aged_out_runs",
+ new_callable=AsyncMock,
+ return_value=[],
+ ),
):
yield
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py
index 310164036f68..487def7575bb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py
@@ -1460,6 +1460,224 @@ class TestReconcileAbandonedRuns:
assert mock_release.call_args.kwargs["token"] == "wf-ab"
[email protected]_db(transaction=True)
+class TestAgedOutRuns:
+ CONSUMER_MOD = TestReconcileAbandonedRuns.CONSUMER_MOD
+
+ async def _insert_aged(
+ self, conn: psycopg.AsyncConnection[Any], *, state: str = "pending", age: str = "7 days", **overrides: Any
+ ) -> str:
+ bid = await _insert_batch(conn, **overrides)
+ if state != "pending":
+ await BatchQueue.update_status(conn, batch_id=bid, job_state=state, attempt=1)
+ await conn.execute(f"UPDATE {BATCH_TABLE} SET created_at = now() - interval '{age}' WHERE id = %s", (bid,))
+ return bid
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "state,age,expected",
+ [
+ ("pending", "7 days", ["run-1"]),
+ ("waiting_retry", "7 days", ["run-1"]),
+ # Still inside the claim window, so a consumer can still load it.
+ ("pending", "6 days", []),
+ ("waiting_retry", "6 days", []),
+ # The stale-executing sweep and the failed-run reconcile own these states.
+ ("executing", "7 days", []),
+ ("failed", "7 days", []),
+ ("succeeded", "7 days", []),
+ ],
+ )
+ async def test_finds_runs_with_a_batch_no_claim_can_reach(self, conn, state, age, expected):
+ await self._insert_aged(conn, state=state, age=age, batch_index=0, metadata={"workflow_run_id": "wf-1"})
+ await _insert_batch(conn, batch_index=1)
+
+ refs = await BatchQueue.get_aged_out_runs(conn, limit=10)
+
+ assert [r.run_uuid for r in refs] == expected
+ if expected:
+ ref = refs[0]
+ assert (ref.job_id, ref.workflow_run_id, ref.first_aged_out_batch_index, ref.aged_out_batches) == (
+ "job-1",
+ "wf-1",
+ 0,
+ 1,
+ )
+
+ @pytest.mark.asyncio
+ async def test_leaves_runs_with_a_failed_batch_to_the_failed_run_reconcile(self, conn):
+ await self._insert_aged(conn, batch_index=0)
+ failed = await _insert_batch(conn, batch_index=1)
+ await BatchQueue.update_status(conn, batch_id=failed, job_state="failed", attempt=1)
+
+ assert await BatchQueue.get_aged_out_runs(conn, limit=10) == []
+
+ @pytest.mark.asyncio
+ async def test_oldest_run_first_so_the_limit_cannot_starve_the_backlog(self, conn):
+ for run_uuid, age in (("run-new", "7 days"), ("run-old", "9 days"), ("run-mid", "8 days")):
+ await self._insert_aged(conn, age=age, run_uuid=run_uuid)
+
+ refs = await BatchQueue.get_aged_out_runs(conn, limit=2)
+
+ assert [r.run_uuid for r in refs] == ["run-old", "run-mid"]
+
+ @pytest.mark.asyncio
+ async def test_candidate_scan_can_use_the_claimable_index(self, conn):
+ from products.warehouse_sources_queue.backend.core.jobs_db import _aged_out_candidate_runs_sql
+
+ await self._insert_aged(conn)
+ await conn.execute("SET enable_seqscan = off")
+ try:
+ cur = await conn.execute("EXPLAIN (FORMAT TEXT) " + _aged_out_candidate_runs_sql(), {"limit": 100})
+ plan = "\n".join(row[0] for row in await cur.fetchall())
+ finally:
+ await conn.execute("SET enable_seqscan = on")
+
+ assert "sb_claimable_idx" in plan
+
+ @pytest.mark.asyncio
+ async def test_reconcile_fails_the_run_so_its_later_batches_never_load(self, conn):
+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer import (
+ AGED_OUT_RUN_ERROR,
+ )
+
+ # A long serialized run: batch 0 aged out while the loader worked other groups, and the
+ # final batch is still fresh enough to claim. Loading it would complete the job.
+ metadata = {"workflow_run_id": "wf-long"}
+ await self._insert_aged(conn, batch_index=0, run_uuid="run-long", job_id="job-long", metadata=metadata)
+ await _insert_batch(
+ conn, batch_index=1, run_uuid="run-long", job_id="job-long", metadata=metadata, is_final_batch=True
+ )
+ await _insert_batch(conn, run_uuid="run-other", job_id="job-other", schema_id="schema-2")
+
+ adapter = DeltaBatchConsumerAdapter()
+ with (
+ patch(f"{self.CONSUMER_MOD}.mark_job_failed_if_not_terminal", return_value=True) as mock_fail_job,
+ patch(f"{self.CONSUMER_MOD}.release_v3_pipeline_lock", return_value=True) as mock_release,
+ ):
+ await adapter.reconcile_failed_runs(conn, grace_seconds=120, lookback_seconds=86400, limit=100)
+
+ rows = await (
+ await conn.execute(
+ f"SELECT latest_state FROM {BATCH_TABLE} WHERE run_uuid = 'run-long' ORDER BY batch_index"
+ )
+ ).fetchall()
+ assert [r[0] for r in rows] == ["failed", "failed"]
+ mock_fail_job.assert_called_once_with(job_id="job-long", team_id=1, error=AGED_OUT_RUN_ERROR)
+ mock_release.assert_called_once_with(team_id=1, schema_id="schema-1", token="wf-long")
+
+ claimed = await _claim(conn, limit=10)
+ assert [b.run_uuid for b in claimed] == ["run-other"]
+ await _release(conn, batches=claimed)
+
+
[email protected]_db(transaction=True)
+class TestUnfinishedEarlierBatchesSync:
+ @pytest.fixture(autouse=True)
+ def _fresh_pool(self):
+ _SYNC_POOLS.clear()
+ yield
+ for pool in _SYNC_POOLS.values():
+ for pooled in pool._idle:
+ pooled.close()
+ _SYNC_POOLS.clear()
+
+ def _check(self, db_url: str, *, before_batch_index: int = 3, run_uuid: str = "run-1"):
+ return BatchQueue.get_unfinished_earlier_batches_sync(
+ db_url,
+ team_id=1,
+ schema_id="schema-1",
+ run_uuid=run_uuid,
+ before_batch_index=before_batch_index,
+ connect_timeout_seconds=5,
+ )
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "earlier_states,expected_count,expected_first,expected_states",
+ [
+ (["succeeded", "succeeded", "succeeded"], 0, None, ()),
+ # An aged-out batch never moves past 'pending'.
+ (["succeeded", "pending", "succeeded"], 1, 1, ("pending",)),
+ (["failed", "succeeded", "waiting_retry"], 2, 0, ("failed", "waiting_retry")),
+ (["succeeded", "succeeded", "executing"], 1, 2, ("executing",)),
+ # A lane leaves gaps in its batch indexes, and a gap is not a missing batch.
+ (["succeeded", None, "succeeded"], 0, None, ()),
+ ],
+ )
+ async def test_reports_earlier_batches_that_did_not_succeed(
+ self, conn, _db_url, earlier_states, expected_count, expected_first, expected_states
+ ):
+ for index, state in enumerate(earlier_states):
+ if state is None:
+ continue
+ bid = await _insert_batch(conn, batch_index=index)
+ if state != "pending":
+ await BatchQueue.update_status(conn, batch_id=bid, job_state=state, attempt=1)
+ # The batches of the write being completed are still executing, and must not count.
+ final = await _insert_batch(conn, batch_index=3, is_final_batch=True)
+ await BatchQueue.update_status(conn, batch_id=final, job_state="executing", attempt=1)
+
+ unfinished = self._check(_db_url)
+
+ assert (unfinished.count, unfinished.first_batch_index, unfinished.states) == (
+ expected_count,
+ expected_first,
+ expected_states,
+ )
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "other",
+ [
+ {"run_uuid": "run-2"},
+ {"schema_id": "schema-2"},
+ {"team_id": 2},
+ ],
+ )
+ async def test_ignores_batches_of_other_runs_and_groups(self, conn, _db_url, other):
+ await _insert_batch(conn, batch_index=0, **other)
+
+ assert self._check(_db_url).count == 0
+
+ @pytest.mark.asyncio
+ async def test_reports_the_aged_out_batch_that_the_claim_skipped(self, conn, _db_url):
+ # The run gates do not read an aged-out 'pending' batch, so the claim hands out the final
+ # batch of its run. The completion check must still see the aged batch.
+ aged = await _insert_batch(conn, batch_index=0)
+ await conn.execute(f"UPDATE {BATCH_TABLE} SET created_at = now() - interval '7 days' WHERE id = %s", (aged,))
+ await _insert_batch(conn, batch_index=1, is_final_batch=True)
+
+ claimed = await _claim(conn)
+ try:
+ assert [b.batch_index for b in claimed] == [1]
+ unfinished = self._check(_db_url, before_batch_index=1)
+ finally:
+ await _release(conn, batches=claimed)
+
+ assert (unfinished.count, unfinished.first_batch_index, unfinished.states) == (1, 0, ("pending",))
+
+ @pytest.mark.asyncio
+ async def test_gives_up_at_its_statement_timeout(self, conn, conn_b, _db_url):
+ await _insert_batch(conn, batch_index=0)
+ # A lock that the check must wait for stands in for partition DDL on the queue DB.
+ async with conn_b.transaction():
+ await conn_b.execute(f"LOCK TABLE {BATCH_TABLE} IN ACCESS EXCLUSIVE MODE")
+ with pytest.raises(psycopg.errors.QueryCanceled):
+ BatchQueue.get_unfinished_earlier_batches_sync(
+ _db_url,
+ team_id=1,
+ schema_id="schema-1",
+ run_uuid="run-1",
+ before_batch_index=3,
+ connect_timeout_seconds=5,
+ statement_timeout_ms=50,
+ )
+
+ # A canceled check leaves the pooled connection usable.
+ assert self._check(_db_url).count == 1
+
+
@pytest.mark.django_db(transaction=True)
class TestClaimWindowSkipsForeignLeasedGroups:
@pytest.mark.asyncio
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/queue_replay.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/queue_replay.py
index 17ff427cdd1f..8bfe759e1428 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/queue_replay.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/queue_replay.py
@@ -22,7 +22,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
process_message,
)
from products.warehouse_sources.backend.types import ExternalDataJobStatus
-from products.warehouse_sources_queue.backend.core.jobs_db import BATCH_TABLE, PendingBatch
+from products.warehouse_sources_queue.backend.core.jobs_db import BATCH_TABLE, PendingBatch, build_status_dual_write_sql
from products.warehouse_sources_queue.backend.testing import ensure_queue_tables, get_test_database_url
PRODUCER_DATABASE_URL_PATH = (
@@ -69,7 +69,15 @@ class PostgresQueueReplay:
try:
process_message(batch.to_export_signal())
except Exception:
- pass
+ continue
+ # The real consumer records each loaded batch, and a final batch completes its job only
+ # when every earlier batch of the run has this state. The write must commit outside the
+ # test transaction, because the loader reads the queue on its own connection.
+ with psycopg.connect(get_test_database_url(), autocommit=True) as queue_conn:
+ queue_conn.execute(
+ build_status_dual_write_sql(with_batch_created_at=False),
+ {"batch_id": row["id"], "job_state": "succeeded", "attempt": 1, "error_response": None},
+ )
def get_run_uuids_for_job(self, job_id: str) -> list[str]:
with django_conn.cursor() as cur:
@@ -155,6 +163,7 @@ async def replay_v3_consumer(
DATA_WAREHOUSE_REDIS_HOST="localhost",
DATA_WAREHOUSE_REDIS_PORT="6379",
DATAWAREHOUSE_BUCKET=bucket_name,
+ WAREHOUSE_SOURCES_DATABASE_URL=get_test_database_url(),
),
mock.patch(
"products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.is_batch_already_processed",
diff --git a/products/warehouse_sources_queue/backend/README.md b/products/warehouse_sources_queue/backend/README.md
index c73d2418c771..dfdc14ab4d7c 100644
--- a/products/warehouse_sources_queue/backend/README.md
+++ b/products/warehouse_sources_queue/backend/README.md
@@ -64,6 +64,9 @@ All SQL lives in `core/jobs_db.py`; the polling/retry/recovery engine is `core/b
It is single-flighted fleet-wide through a sentinel lease row (`try_acquire_reconcile_sweep_slot`, team_id 0 / `__reconcile-sweep__`, 240s slot TTL): the sweeps reconcile global state, so N pods running them concurrently is pure duplicated load, exactly when the queue DB can least afford it.
- **Stranded-run sweep** (same cadence and connection, `get_stale_stranded_runs`): runs the loader abandoned, meaning non-terminal batches, no live lease, and no loader progress for 6 hours.
These have no `failed` batch for the reconcile sweep to key on (the extraction died before a final batch), so without this pass they would strand until the retention prune.
+ - **Aged-out run sweep** (same cadence and connection, `get_aged_out_runs`): runs that hold a `pending` or `waiting_retry` batch older than `CLAIM_ELIGIBILITY_INTERVAL`.
+ No claim offers that batch again, but the run gates do not read `pending` rows, so the later batches of the run stay claimable. The sweep fails the run's batches, then the job, then releases the pipeline lock.
+ The stranded-run sweep does not catch these runs, because the group makes progress on the later batches.
- **Sync producer** (`pipeline_v3/postgres_queue/producer.py`, in warehouse_sources): runs inside Temporal activities, plain `psycopg.Connection` with autocommit. Each queue row is a single INSERT.
The v3 pipeline holds each batch's row back by one (`hold_batch`): staging batch N inserts the row for batch N-1, and the end of the run inserts the held row with `is_final_batch = true`, so the run's last data row is its own final marker and the loader reads every parquet file once.
A resumable source releases the held row before it commits a cursor (`release_held_batch`), because the cursor promises that every row before it is loadable; when that leaves nothing held at the end, `send_final_batch` inserts the last batch a second time as a final-only marker, which the loader still accepts (`_run_post_load_for_already_processed_batch`).
@@ -82,6 +85,7 @@ All SQL lives in `core/jobs_db.py`; the polling/retry/recovery engine is `core/b
The bucket's 8-day lifecycle rule for extraction files also stays above this window, so a claimable batch's parquet still exists.
`TestClaimEligibilityWindow` in `products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py` enforces the retention coupling.
The bucket rule lives in the infrastructure repo, and nothing in this repo checks it.
+ As a second guard, the loader refuses to complete a job when an earlier batch of the final batch's run did not reach `succeeded` (`_refuse_incomplete_run` in `load/processor.py`). The consumer then fails the run, so the staged cursor never moves past rows that did not load.
- **Partition pruning bounds**: consumer queries include `created_at > now() - interval '14 days'` (2x the retention) so the planner can skip dropped partitions.
### Query cost scales with the answer, not with history
@@ -129,7 +133,7 @@ The overall topology (Temporal producer activity, two tables, a fleet of consume
The consumer engine emits nine `warehouse_pg_consumer_*` metrics (`metrics.py`): `batches_processed_total`, `batch_processing_duration_seconds`, `batch_retry_total`, `runs_failed_total`, `poll_duration_seconds`, `poll_batches_fetched`, `poll_failures_total`, `active_groups`, `recovery_sweeps_total`.
Other sinks get that nine-metric engine set under their own `{prefix}_pg_consumer_*` names via `make_consumer_metrics`, so dashboards and KEDA queries never conflate two consumers' series.
-Two further counters exist only under the `warehouse_pg_consumer_*` prefix, incremented by the Delta consumer's reconcile sweep (`consumer.py`) rather than by the shared engine: `runs_reconciled_total` and `runs_terminalized_stale_total`.
+Further counters exist only under the `warehouse_pg_consumer_*` prefix, incremented by the Delta consumer's reconcile sweep (`consumer.py`) rather than by the shared engine: `runs_reconciled_total`, `runs_terminalized_stale_total` and `runs_failed_aged_out_total`.
The headline health signal is `warehouse_pg_queue_oldest_unclaimed_batch_seconds`: the age of the oldest batch no consumer has picked up yet.
It is the loader's data-freshness signal and rises whenever loading stalls, regardless of cause, so its alert fires even when every other signal looks green.
diff --git a/products/warehouse_sources_queue/backend/core/jobs_db.py b/products/warehouse_sources_queue/backend/core/jobs_db.py
index 5dd855cfa2bf..f693017e6de9 100644
--- a/products/warehouse_sources_queue/backend/core/jobs_db.py
+++ b/products/warehouse_sources_queue/backend/core/jobs_db.py
@@ -99,7 +99,8 @@ PARTITION_PRUNING_INTERVAL = "14 days"
# batch's file still exists. Applies only to the outer claim
# candidates and the recovery sweep — never to the head-of-line/failed-run/
# schema-busy gates, which must keep seeing rows aged past this window for as
-# long as they exist.
+# long as they exist. A 'pending' or 'waiting_retry' batch past this window can
+# never load, so the reconcile sweep fails its run (see `get_aged_out_runs`).
CLAIM_ELIGIBILITY_INTERVAL = "6 days 12 hours"
# One claim reads a window of the queue, not the whole claimable set, so its cost
@@ -935,6 +936,71 @@ def _stranded_candidate_runs_sql() -> str:
"""
+def _aged_out_candidate_runs_sql() -> str:
+ """Runs that hold a 'pending' or 'waiting_retry' batch older than ``CLAIM_ELIGIBILITY_INTERVAL``.
+
+ The claim query never offers such a batch again, and the run gates do not
+ read 'pending' rows. So the later batches of the run stay claimable, and
+ the run can load past the batch that will never load. The reconcile sweep
+ fails these runs before their final batch can complete the job.
+
+ The candidate states are exactly ``sb_claimable_idx``'s, for the same reason
+ as in :func:`_orphaned_candidate_runs_sql`. Runs that hold a 'failed' batch
+ are left to the failed-run reconcile and the orphan drain. The ``OFFSET 0``
+ fence keeps that gate one ``sb_run_gate_idx`` probe per candidate run.
+ """
+ return f"""
+ WITH aged_runs AS (
+ SELECT
+ b.run_uuid,
+ b.team_id,
+ b.schema_id,
+ MAX(b.job_id) AS job_id,
+ MAX(b.metadata->>'workflow_run_id') AS workflow_run_id,
+ MIN(b.created_at) AS oldest_created_at,
+ MIN(b.batch_index) AS first_aged_out_batch_index,
+ count(*) AS aged_out_batches
+ FROM {BATCH_TABLE} b
+ WHERE b.created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'
+ AND b.created_at <= now() - interval '{CLAIM_ELIGIBILITY_INTERVAL}'
+ AND b.latest_state IN ('pending', 'waiting_retry')
+ GROUP BY b.run_uuid, b.team_id, b.schema_id
+ )
+ SELECT
+ r.run_uuid, r.team_id, r.schema_id, r.job_id, r.workflow_run_id,
+ r.first_aged_out_batch_index, r.aged_out_batches
+ FROM aged_runs r
+ WHERE NOT EXISTS (
+ SELECT 1
+ FROM {BATCH_TABLE} bf
+ WHERE bf.run_uuid = r.run_uuid
+ AND bf.created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'
+ AND bf.latest_state = 'failed'
+ OFFSET 0
+ )
+ ORDER BY r.oldest_created_at ASC
+ LIMIT %(limit)s
+ """
+
+
+# Server-side ceiling for the completion check of a final batch. A lock wait (for example behind
+# partition DDL) then fails the attempt, and the batch retries, instead of holding the loader thread.
+UNFINISHED_EARLIER_BATCHES_STATEMENT_TIMEOUT_MS = 30_000
+
+# Reads the earlier batches of one run through ``sb_run_uuid_bi_idx``. The team and
+# schema filters guard against a run_uuid collision across groups.
+_UNFINISHED_EARLIER_BATCHES_SQL = f"""
+ SELECT count(*), MIN(b.batch_index), COALESCE(array_agg(DISTINCT b.latest_state), '{{}}')
+ FROM {BATCH_TABLE} b
+ WHERE b.run_uuid = %(run_uuid)s
+ AND b.batch_index < %(before_batch_index)s
+ AND b.team_id = %(team_id)s
+ AND b.schema_id = %(schema_id)s
+ AND b.created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'
+ AND b.latest_state <> 'succeeded'
+"""
+
+
def _stale_executing_sql(scope_sql: str = "") -> str:
"""Shared body of the stale-executing sweep (async consumer and its sync ops twin).
@@ -1159,6 +1225,33 @@ class StrandedRunRef:
non_terminal_batches: int
+@dataclass(frozen=True, slots=True)
+class AgedOutRunRef:
+ """A run with a batch that aged out of ``CLAIM_ELIGIBILITY_INTERVAL`` before it loaded.
+
+ The batch will never load, because no claim offers it again and its parquet
+ goes with the bucket lifecycle rule. The run can therefore never complete
+ correctly, and the reconcile sweep fails it.
+ """
+
+ run_uuid: str
+ job_id: str
+ team_id: int
+ schema_id: str
+ workflow_run_id: str | None
+ first_aged_out_batch_index: int
+ aged_out_batches: int
+
+
+@dataclass(frozen=True, slots=True)
+class UnfinishedEarlierBatches:
+ """The batches of a run, before a given batch_index, whose latest state is not 'succeeded'."""
+
+ count: int
+ first_batch_index: int | None
+ states: tuple[str, ...]
+
+
@dataclass(frozen=True, slots=True)
class OrphanedRunRef:
"""A run holding a ``failed`` batch that still has non-terminal batches behind it.
@@ -1957,6 +2050,55 @@ class BatchQueue:
raise
raise AssertionError("unreachable")
+ @staticmethod
+ def get_unfinished_earlier_batches_sync(
+ database_url: str,
+ *,
+ team_id: int,
+ schema_id: str,
+ run_uuid: str,
+ before_batch_index: int,
+ connect_timeout_seconds: int = 10,
+ statement_timeout_ms: int = UNFINISHED_EARLIER_BATCHES_STATEMENT_TIMEOUT_MS,
+ ) -> UnfinishedEarlierBatches:
+ """The batches of a run before ``before_batch_index`` that did not reach 'succeeded'.
+
+ The loader asks this before a final batch completes its job. A batch that
+ a run skipped (aged out, failed, still queued) means that the run is not
+ complete, so the staged cursor must not advance past it. Runs on the same
+ small sync pool as :meth:`verify_group_lease_sync`, with the same retry for a
+ connection the queue DB dropped while idle.
+
+ Raises ``psycopg.errors.QueryCanceled`` past ``statement_timeout_ms``.
+ """
+ pool = _sync_connection_pool(database_url, connect_timeout_seconds)
+ params = {
+ "team_id": team_id,
+ "schema_id": schema_id,
+ "run_uuid": run_uuid,
+ "before_batch_index": before_batch_index,
+ }
+ for attempt in (1, 2):
+ with pool.connection() as conn:
+ try:
+ # The timeout is transaction-local, so it never leaks into the pooled session.
+ with conn.transaction():
+ conn.execute("SELECT set_config('statement_timeout', %s, true)", (str(statement_timeout_ms),))
+ row = conn.execute(_UNFINISHED_EARLIER_BATCHES_SQL, params).fetchone()
+ except psycopg.errors.QueryCanceled:
+ raise
+ except psycopg.OperationalError:
+ pool.discard(conn)
+ if attempt == 2:
+ raise
+ continue
+ assert row is not None
+ count, first_batch_index, states = row
+ return UnfinishedEarlierBatches(
+ count=count, first_batch_index=first_batch_index, states=tuple(sorted(states))
+ )
+ raise AssertionError("unreachable")
+
@staticmethod
async def get_stale_executing(
conn: psycopg.AsyncConnection[Any],
@@ -2358,6 +2500,35 @@ class BatchQueue:
for row in rows
]
+ @staticmethod
+ async def get_aged_out_runs(
+ conn: psycopg.AsyncConnection[Any],
+ *,
+ limit: int,
+ ) -> list[AgedOutRunRef]:
+ """Runs with a non-terminal batch past ``CLAIM_ELIGIBILITY_INTERVAL``, oldest first.
+
+ The stranded-run sweep cannot catch these. It spares a group that makes
+ progress, and a long serialized run makes progress on its later batches
+ while an earlier one ages out. See :func:`_aged_out_candidate_runs_sql`.
+ """
+ async with conn.cursor(row_factory=dict_row) as cur:
+ await cur.execute(_aged_out_candidate_runs_sql(), {"limit": limit})
+ rows = await cur.fetchall()
+
+ return [
+ AgedOutRunRef(
+ run_uuid=row["run_uuid"],
+ job_id=row["job_id"],
+ team_id=row["team_id"],
+ schema_id=row["schema_id"],
+ workflow_run_id=row["workflow_run_id"],
+ first_aged_out_batch_index=row["first_aged_out_batch_index"],
+ aged_out_batches=row["aged_out_batches"],
+ )
+ for row in rows
+ ]
+
@staticmethod
async def get_queue_freshness(
conn: psycopg.AsyncConnection[Any],
diff --git a/products/warehouse_sources_queue/backend/core/metrics.py b/products/warehouse_sources_queue/backend/core/metrics.py
index 503e2067c660..05bdcaca3087 100644
--- a/products/warehouse_sources_queue/backend/core/metrics.py
+++ b/products/warehouse_sources_queue/backend/core/metrics.py
@@ -107,6 +107,13 @@ RUNS_TERMINALIZED_STALE_TOTAL = Counter(
"staleness threshold) that the reconcile sweep failed before the retention prune would",
)
+RUNS_FAILED_AGED_OUT_TOTAL = Counter(
+ "warehouse_pg_consumer_runs_failed_aged_out_total",
+ "Runs the reconcile sweep failed because a batch aged out of the claim window before it "
+ "loaded. Without the sweep, the later batches of such a run could complete the job past "
+ "the batch that never loaded",
+)
+
# The loader's data-freshness signal: it rises whenever loading stalls,
# regardless of why (wedged consumers, claim-query degradation, crashloops).
# One elected pod per fleet samples it (see clear_queue_sample_gauges), so