:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): fail runs whose batches age out of the claim window (#119691)
Tom Owers1aee83ac6f8b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.pyindex ef9989b6b3f6..b361680ca6c5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/metrics.py@@ -17,6 +17,11 @@ IDEMPOTENCY_HIT_TOTAL = Counter("Total idempotency cache hits (batch already processed)",)+INCOMPLETE_RUN_COMPLETIONS_REFUSED_TOTAL = Counter(+ "warehouse_load_incomplete_run_completions_refused_total",+ "Final batches that did not complete their job because an earlier batch of the run did not load",+)+APPEND_ROLLBACK_TOTAL = Counter("warehouse_load_append_rollback_total","Append rollback attempts by outcome",diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.pyindex 07dce69f7c86..894839edfcde 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py@@ -96,6 +96,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipelineDELTA_ROWS_WRITTEN_TOTAL,DELTA_WRITE_DURATION_SECONDS,IDEMPOTENCY_HIT_TOTAL,+ INCOMPLETE_RUN_COMPLETIONS_REFUSED_TOTAL,PARQUET_READ_DURATION_SECONDS,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.table_handles import (@@ -121,6 +122,7 @@ from products.warehouse_sources.backend.temporal.data_imports.util import preparfrom products.warehouse_sources.backend.temporal.data_imports.workload_report import report_phase, workload_reportingfrom products.warehouse_sources.backend.types import ExternalDataSourceTypefrom products.warehouse_sources_queue.backend.core.batch_consumer import CoalescingDeclined, OwnershipLostError+from products.warehouse_sources_queue.backend.core.jobs_db import BatchQueuelogger = structlog.get_logger(__name__)@@ -132,6 +134,20 @@ FILE_COUNT_LOG_SAMPLE_EVERY = 100REBUILD_COMPACT_CHECK_EVERY = 10REBUILD_COMPACTION_FLAG = "warehouse-rebuild-run-compaction"+# Shown to the customer verbatim as latest_error. The consumer also matches it to fail the run+# without retries, because the earlier batches cannot load on a later attempt of this run.+INCOMPLETE_RUN_ERROR = (+ "Some batches of this sync run did not load, so the sync did not complete. "+ "The next scheduled sync retries automatically. No action is needed."+)+++class IncompleteRunError(Exception):+ """A final batch arrived while earlier batches of its run had not loaded."""++ def __init__(self) -> None:+ super().__init__(INCOMPLETE_RUN_ERROR)+def _get_write_type(sync_type: SyncTypeLiteral) -> Literal["incremental", "full_refresh", "append"]:"""Convert sync type to write type for DeltaTableRef."""@@ -546,7 +562,45 @@ def _release_pipeline_lock_for_job(export_signal: ExportSignalMessage) -> None:capture_exception(e)+def _refuse_incomplete_run(export_signal: ExportSignalMessage) -> None:+ """Raise when an earlier batch of the run did not load, before the job completes.++ The claim gates do not see a batch that aged out of the claim window, so the later batches of+ its run can still load. Completing such a run would promote the staged cursor past rows that+ never loaded, and the next sync would never read them again. A failed job keeps the committed+ cursor, so the next sync extracts those rows again.++ ``batch_index`` is the first batch this write loaded for the run, so every batch of the run+ below it loaded in an earlier write.+ """+ unfinished = BatchQueue.get_unfinished_earlier_batches_sync(+ settings.WAREHOUSE_SOURCES_DATABASE_URL,+ team_id=export_signal.team_id,+ schema_id=export_signal.schema_id,+ run_uuid=export_signal.run_uuid,+ before_batch_index=export_signal.batch_index,+ )+ if unfinished.count == 0:+ return++ INCOMPLETE_RUN_COMPLETIONS_REFUSED_TOTAL.inc()+ logger.error(+ "run_completion_refused_unfinished_batches",+ team_id=export_signal.team_id,+ external_data_schema_id=export_signal.schema_id,+ external_data_job_id=export_signal.job_id,+ run_uuid=export_signal.run_uuid,+ before_batch_index=export_signal.batch_index,+ unfinished_batches=unfinished.count,+ first_unfinished_batch_index=unfinished.first_batch_index,+ unfinished_states=list(unfinished.states),+ )+ raise IncompleteRunError()++def _mark_job_completed(export_signal: ExportSignalMessage) -> None:+ _refuse_incomplete_run(export_signal)+# Reconnect stale connections before the transaction; close_old_connections must never# run inside an atomic block since it can drop the connection mid-transaction.close_old_connections()diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.pyindex e2bb8d598a33..258d0f773a92 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py@@ -37,6 +37,8 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.pospost_load_phase,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor import (+ INCOMPLETE_RUN_ERROR,+ IncompleteRunError,_apply_partitioning,_enrich_cdc_rows,_file_count_after_write,@@ -58,6 +60,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipelinefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.test_mocks import mock_delta_tablefrom products.warehouse_sources.backend.types import ExternalDataSourceTypefrom products.warehouse_sources_queue.backend.core.batch_consumer import CoalescingDeclined+from products.warehouse_sources_queue.backend.core.jobs_db import UnfinishedEarlierBatches++_COMPLETE_RUN = UnfinishedEarlierBatches(count=0, first_batch_index=None, states=())@pytest.fixture(autouse=True)@@ -71,6 +76,16 @@ def _no_close_old_connections():yield[email protected](autouse=True)+def _unfinished_earlier_batches() -> Iterator[MagicMock]:+ # The completion check reads the queue DB, which these unit tests do not have.+ with patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.BatchQueue.get_unfinished_earlier_batches_sync",+ return_value=_COMPLETE_RUN,+ ) as check:+ yield check++@pytest.fixture(autouse=True)def _table_handles() -> Iterator[GroupTableHandles]:# The loader's registry is process-wide, so a handle that one test left would reach the next.@@ -524,6 +539,50 @@ class TestMarkJobCompleted:mock_finish_row_tracking.assert_not_awaited()mock_release.assert_not_called()+ @parameterized.expand(+ [+ (+ "aged_out_batch_never_loaded",+ UnfinishedEarlierBatches(count=1, first_batch_index=2, states=("pending",)),+ ),+ ("earlier_batch_failed", UnfinishedEarlierBatches(count=1, first_batch_index=0, states=("failed",))),+ (+ "several_batches_left",+ UnfinishedEarlierBatches(count=3, first_batch_index=1, states=("pending", "waiting_retry")),+ ),+ ]+ )+ @patch(f"{_PROCESSOR}.release_v3_pipeline_lock")+ @patch(f"{_PROCESSOR}.ExternalDataSchema.objects")+ @patch(f"{_PROCESSOR}.update_external_job_status")+ def test_a_run_with_an_unfinished_earlier_batch_does_not_complete(+ self,+ _case: str,+ unfinished: UnfinishedEarlierBatches,+ mock_update: MagicMock,+ mock_schema_objects: MagicMock,+ mock_release: MagicMock,+ ) -> None:+ signal = self._make_signal()+ signal.batch_index = 5++ with (+ patch(f"{_PROCESSOR}.BatchQueue.get_unfinished_earlier_batches_sync", return_value=unfinished) as check,+ pytest.raises(IncompleteRunError, match=INCOMPLETE_RUN_ERROR),+ ):+ _mark_job_completed(signal)++ assert check.call_args.kwargs == {+ "team_id": 1,+ "schema_id": "schema-1",+ "run_uuid": "run-abc-a1",+ "before_batch_index": 5,+ }+ mock_update.assert_not_called()+ mock_schema_objects.get.return_value.promote_staged_incremental_values.assert_not_called()+ # The consumer fails the run on this error, and its fail path releases the lock.+ mock_release.assert_not_called()+class TestRedeliveredFinalBatchPostLoad:@parameterized.expand([("companion", "scd2_append"), ("consolidated", "incremental_merge"), ("snapshot", None)])@@ -824,6 +883,9 @@ class TestProcessMessages:# Without the flag the post-import workflow opens the table again to measure a size that# post-load already recorded.mock_trigger.assert_called_once_with(ANY, True)+ # The completion check counts the batches below this index as earlier writes. The tail index+ # would count the set's own members, which are still executing, and fail every coalesced run.+ assert mock_mark_completed.call_args.args[0].batch_index == 0@patch(f"{_PROCESSOR}.posthoganalytics")@patch(f"{_PROCESSOR}.read_parquet")diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.pyindex 0a4d5e78b729..3b263cd69f07 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/consumer.py@@ -45,6 +45,9 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipelinefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.destinations_load.errors import (DESTINATION_CONFIGURATION_ERROR_MARKER,)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor import (+ INCOMPLETE_RUN_ERROR,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.table_handles import (release_group_table_handle,)@@ -74,6 +77,7 @@ from products.warehouse_sources_queue.backend.core.jobs_db import (FRESHNESS_WINDOW_SECONDS,GAUGE_STATEMENT_TIMEOUT_MS,TAKEOVER_STALE_THRESHOLD_SECONDS,+ AgedOutRunRef,BatchQueue,ClaimCursor,ExecutingAttempt,@@ -94,6 +98,7 @@ from products.warehouse_sources_queue.backend.core.metrics import (DRAINED_AFTER_FAILURE_TOTAL,OLDEST_UNCLAIMED_BATCH_SECONDS,ORPHANED_BATCHES_DRAINED_TOTAL,+ RUNS_FAILED_AGED_OUT_TOTAL,RUNS_RECONCILED_TOTAL,RUNS_TERMINALIZED_STALE_TOTAL,SERIALIZED_BATCHES,@@ -134,6 +139,13 @@ STRANDED_RUN_ERROR = ("The next scheduled sync retries automatically. No action is needed.")+# Stamped on a run with a batch that aged out of the claim window before it loaded. Shown to the+# customer verbatim as latest_error, like STRANDED_RUN_ERROR.+AGED_OUT_RUN_ERROR = (+ "Part of this sync run waited too long to load, so the sync did not complete. "+ "The next scheduled sync retries automatically. No action is needed."+)+# Stamped on batches retired by the orphan drain. Their run already failed and the customer has# already seen that failure on the job, so this text only has to explain the batch rows themselves.ORPHANED_BATCH_ERROR = "left over from a run that had already failed (orphan drain)"@@ -185,6 +197,8 @@ NON_RETRYABLE_ERROR_PATTERNS: tuple[str, ...] = (# a destination's own settings refuse the connection (bad credentials, unknown database,# unroutable host); the next scheduled run tries again after the customer fixes themDESTINATION_CONFIGURATION_ERROR_MARKER,+ # an earlier batch of the run did not load, and a retry of the final batch cannot load it+ INCOMPLETE_RUN_ERROR,)# Subset of the non-retryable errors that are expected upstream/customer conditions rather than@@ -567,6 +581,20 @@ class DeltaBatchConsumerAdapter:logger.exception("orphaned_batch_drain_failed")capture_exception(e)+ # Before the stranded pass, so that a run with an aged-out batch gets this reason. Isolated so+ # its failure can't take the sweep down.+ try:+ await self._fail_aged_out_runs(conn, limit=limit)+ except psycopg.OperationalError as e:+ if _is_transient_queue_connection_drop(e, conn):+ logger.warning("aged_out_run_sweep_closed_connection", error=str(e))+ else:+ logger.exception("aged_out_run_sweep_failed")+ capture_exception(e)+ except Exception as e:+ logger.exception("aged_out_run_sweep_failed")+ capture_exception(e)+# Runs the loader abandoned leave no 'failed' batch for get_failed_runs to key on, so sweep# them on the same cadence and connection. Isolated so its failure can't take the sweep down.try:@@ -701,6 +729,91 @@ class DeltaBatchConsumerAdapter:)capture_exception(e)+ async def _fail_aged_out_runs(self, conn: psycopg.AsyncConnection[Any], *, limit: int) -> None:+ """Fail runs with a batch that aged out of the claim window before it loaded.++ The batch never loads, but the run gates do not see it, so the later batches of the run stay+ claimable. Without this pass the final batch could complete the job and promote the cursor+ past the rows that never loaded. The stranded-run pass does not catch it, because the group+ makes progress on those later batches.++ Batches are failed FIRST, for the same reason as in ``_reconcile_stale_stranded_runs``: the+ later batches are still claimable, and a crash after this write leaves a failed batch for+ ``reconcile_failed_runs`` to finalize the job from.+ """+ with observe_queue_query("get_aged_out_runs"):+ refs = await BatchQueue.get_aged_out_runs(conn, limit=limit)+ for ref in refs:+ try:+ failed_batches = await BatchQueue.fail_run(+ conn,+ run_uuid=ref.run_uuid,+ team_id=ref.team_id,+ schema_id=ref.schema_id,+ reason=AGED_OUT_RUN_ERROR,+ )+ except Exception as e:+ if _is_transient_queue_connection_drop(e, conn):+ raise+ logger.exception("aged_out_run_batch_sweep_failed", run_uuid=ref.run_uuid)+ capture_exception(e)+ continue++ job_reconciled = await self._mark_aged_out_job_failed(ref)+ RUNS_FAILED_AGED_OUT_TOTAL.inc()+ logger.warning(+ "aged_out_run_failed",+ job_id=ref.job_id,+ run_uuid=ref.run_uuid,+ team_id=ref.team_id,+ external_data_schema_id=ref.schema_id,+ first_aged_out_batch_index=ref.first_aged_out_batch_index,+ aged_out_batches=ref.aged_out_batches,+ failed_batches=failed_batches,+ job_reconciled=job_reconciled,+ )++ if ref.workflow_run_id:+ try:+ await sync_to_async(release_v3_pipeline_lock)(+ team_id=ref.team_id,+ schema_id=ref.schema_id,+ token=ref.workflow_run_id,+ )+ except Exception as e:+ logger.error(+ "failed_to_release_v3_pipeline_lock",+ job_id=ref.job_id,+ schema_id=ref.schema_id,+ exc_info=True,+ )+ capture_exception(e)++ async def _mark_aged_out_job_failed(self, ref: AgedOutRunRef) -> bool:+ """Write the run's failure to the app DB. Returns whether this call moved the job to Failed.++ A miss is safe: the run's batches are already failed, so ``reconcile_failed_runs`` finishes+ the job on a later cycle.+ """+ try:+ return await sync_to_async(mark_job_failed_if_not_terminal)(+ job_id=ref.job_id,+ team_id=ref.team_id,+ error=AGED_OUT_RUN_ERROR,+ )+ except Exception as e:+ if is_transient_internal_db_error(e):+ logger.warning(+ "aged_out_run_job_status_update_app_db_not_ready",+ job_id=ref.job_id,+ run_uuid=ref.run_uuid,+ error=str(e),+ )+ else:+ logger.exception("aged_out_run_job_status_update_failed", job_id=ref.job_id, run_uuid=ref.run_uuid)+ capture_exception(e)+ return False+async def _reconcile_stale_stranded_runs(self,conn: psycopg.AsyncConnection[Any],diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.pyindex e7d2a9897d75..4b7b90eac27f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_consumer.py@@ -2518,10 +2518,18 @@ class TestReconcileFailedRuns:@pytest.fixture(autouse=True)def _hold_sweep_slot(self):"""Grant the fleet-wide sweep slot by default — these tests exercise the sweep body, not the slot."""- with patch(- "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.try_acquire_reconcile_sweep_slot",- new_callable=AsyncMock,- return_value=True,+ with (+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.try_acquire_reconcile_sweep_slot",+ new_callable=AsyncMock,+ return_value=True,+ ),+ # The aged-out pass has its own tests against a real queue in test_jobs_db.py.+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer.BatchQueue.get_aged_out_runs",+ new_callable=AsyncMock,+ return_value=[],+ ),):yielddiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.pyindex 310164036f68..487def7575bb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py@@ -1460,6 +1460,224 @@ class TestReconcileAbandonedRuns:assert mock_release.call_args.kwargs["token"] == "wf-ab"[email protected]_db(transaction=True)+class TestAgedOutRuns:+ CONSUMER_MOD = TestReconcileAbandonedRuns.CONSUMER_MOD++ async def _insert_aged(+ self, conn: psycopg.AsyncConnection[Any], *, state: str = "pending", age: str = "7 days", **overrides: Any+ ) -> str:+ bid = await _insert_batch(conn, **overrides)+ if state != "pending":+ await BatchQueue.update_status(conn, batch_id=bid, job_state=state, attempt=1)+ await conn.execute(f"UPDATE {BATCH_TABLE} SET created_at = now() - interval '{age}' WHERE id = %s", (bid,))+ return bid++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "state,age,expected",+ [+ ("pending", "7 days", ["run-1"]),+ ("waiting_retry", "7 days", ["run-1"]),+ # Still inside the claim window, so a consumer can still load it.+ ("pending", "6 days", []),+ ("waiting_retry", "6 days", []),+ # The stale-executing sweep and the failed-run reconcile own these states.+ ("executing", "7 days", []),+ ("failed", "7 days", []),+ ("succeeded", "7 days", []),+ ],+ )+ async def test_finds_runs_with_a_batch_no_claim_can_reach(self, conn, state, age, expected):+ await self._insert_aged(conn, state=state, age=age, batch_index=0, metadata={"workflow_run_id": "wf-1"})+ await _insert_batch(conn, batch_index=1)++ refs = await BatchQueue.get_aged_out_runs(conn, limit=10)++ assert [r.run_uuid for r in refs] == expected+ if expected:+ ref = refs[0]+ assert (ref.job_id, ref.workflow_run_id, ref.first_aged_out_batch_index, ref.aged_out_batches) == (+ "job-1",+ "wf-1",+ 0,+ 1,+ )++ @pytest.mark.asyncio+ async def test_leaves_runs_with_a_failed_batch_to_the_failed_run_reconcile(self, conn):+ await self._insert_aged(conn, batch_index=0)+ failed = await _insert_batch(conn, batch_index=1)+ await BatchQueue.update_status(conn, batch_id=failed, job_state="failed", attempt=1)++ assert await BatchQueue.get_aged_out_runs(conn, limit=10) == []++ @pytest.mark.asyncio+ async def test_oldest_run_first_so_the_limit_cannot_starve_the_backlog(self, conn):+ for run_uuid, age in (("run-new", "7 days"), ("run-old", "9 days"), ("run-mid", "8 days")):+ await self._insert_aged(conn, age=age, run_uuid=run_uuid)++ refs = await BatchQueue.get_aged_out_runs(conn, limit=2)++ assert [r.run_uuid for r in refs] == ["run-old", "run-mid"]++ @pytest.mark.asyncio+ async def test_candidate_scan_can_use_the_claimable_index(self, conn):+ from products.warehouse_sources_queue.backend.core.jobs_db import _aged_out_candidate_runs_sql++ await self._insert_aged(conn)+ await conn.execute("SET enable_seqscan = off")+ try:+ cur = await conn.execute("EXPLAIN (FORMAT TEXT) " + _aged_out_candidate_runs_sql(), {"limit": 100})+ plan = "\n".join(row[0] for row in await cur.fetchall())+ finally:+ await conn.execute("SET enable_seqscan = on")++ assert "sb_claimable_idx" in plan++ @pytest.mark.asyncio+ async def test_reconcile_fails_the_run_so_its_later_batches_never_load(self, conn):+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.postgres_queue.consumer import (+ AGED_OUT_RUN_ERROR,+ )++ # A long serialized run: batch 0 aged out while the loader worked other groups, and the+ # final batch is still fresh enough to claim. Loading it would complete the job.+ metadata = {"workflow_run_id": "wf-long"}+ await self._insert_aged(conn, batch_index=0, run_uuid="run-long", job_id="job-long", metadata=metadata)+ await _insert_batch(+ conn, batch_index=1, run_uuid="run-long", job_id="job-long", metadata=metadata, is_final_batch=True+ )+ await _insert_batch(conn, run_uuid="run-other", job_id="job-other", schema_id="schema-2")++ adapter = DeltaBatchConsumerAdapter()+ with (+ patch(f"{self.CONSUMER_MOD}.mark_job_failed_if_not_terminal", return_value=True) as mock_fail_job,+ patch(f"{self.CONSUMER_MOD}.release_v3_pipeline_lock", return_value=True) as mock_release,+ ):+ await adapter.reconcile_failed_runs(conn, grace_seconds=120, lookback_seconds=86400, limit=100)++ rows = await (+ await conn.execute(+ f"SELECT latest_state FROM {BATCH_TABLE} WHERE run_uuid = 'run-long' ORDER BY batch_index"+ )+ ).fetchall()+ assert [r[0] for r in rows] == ["failed", "failed"]+ mock_fail_job.assert_called_once_with(job_id="job-long", team_id=1, error=AGED_OUT_RUN_ERROR)+ mock_release.assert_called_once_with(team_id=1, schema_id="schema-1", token="wf-long")++ claimed = await _claim(conn, limit=10)+ assert [b.run_uuid for b in claimed] == ["run-other"]+ await _release(conn, batches=claimed)++[email protected]_db(transaction=True)+class TestUnfinishedEarlierBatchesSync:+ @pytest.fixture(autouse=True)+ def _fresh_pool(self):+ _SYNC_POOLS.clear()+ yield+ for pool in _SYNC_POOLS.values():+ for pooled in pool._idle:+ pooled.close()+ _SYNC_POOLS.clear()++ def _check(self, db_url: str, *, before_batch_index: int = 3, run_uuid: str = "run-1"):+ return BatchQueue.get_unfinished_earlier_batches_sync(+ db_url,+ team_id=1,+ schema_id="schema-1",+ run_uuid=run_uuid,+ before_batch_index=before_batch_index,+ connect_timeout_seconds=5,+ )++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "earlier_states,expected_count,expected_first,expected_states",+ [+ (["succeeded", "succeeded", "succeeded"], 0, None, ()),+ # An aged-out batch never moves past 'pending'.+ (["succeeded", "pending", "succeeded"], 1, 1, ("pending",)),+ (["failed", "succeeded", "waiting_retry"], 2, 0, ("failed", "waiting_retry")),+ (["succeeded", "succeeded", "executing"], 1, 2, ("executing",)),+ # A lane leaves gaps in its batch indexes, and a gap is not a missing batch.+ (["succeeded", None, "succeeded"], 0, None, ()),+ ],+ )+ async def test_reports_earlier_batches_that_did_not_succeed(+ self, conn, _db_url, earlier_states, expected_count, expected_first, expected_states+ ):+ for index, state in enumerate(earlier_states):+ if state is None:+ continue+ bid = await _insert_batch(conn, batch_index=index)+ if state != "pending":+ await BatchQueue.update_status(conn, batch_id=bid, job_state=state, attempt=1)+ # The batches of the write being completed are still executing, and must not count.+ final = await _insert_batch(conn, batch_index=3, is_final_batch=True)+ await BatchQueue.update_status(conn, batch_id=final, job_state="executing", attempt=1)++ unfinished = self._check(_db_url)++ assert (unfinished.count, unfinished.first_batch_index, unfinished.states) == (+ expected_count,+ expected_first,+ expected_states,+ )++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "other",+ [+ {"run_uuid": "run-2"},+ {"schema_id": "schema-2"},+ {"team_id": 2},+ ],+ )+ async def test_ignores_batches_of_other_runs_and_groups(self, conn, _db_url, other):+ await _insert_batch(conn, batch_index=0, **other)++ assert self._check(_db_url).count == 0++ @pytest.mark.asyncio+ async def test_reports_the_aged_out_batch_that_the_claim_skipped(self, conn, _db_url):+ # The run gates do not read an aged-out 'pending' batch, so the claim hands out the final+ # batch of its run. The completion check must still see the aged batch.+ aged = await _insert_batch(conn, batch_index=0)+ await conn.execute(f"UPDATE {BATCH_TABLE} SET created_at = now() - interval '7 days' WHERE id = %s", (aged,))+ await _insert_batch(conn, batch_index=1, is_final_batch=True)++ claimed = await _claim(conn)+ try:+ assert [b.batch_index for b in claimed] == [1]+ unfinished = self._check(_db_url, before_batch_index=1)+ finally:+ await _release(conn, batches=claimed)++ assert (unfinished.count, unfinished.first_batch_index, unfinished.states) == (1, 0, ("pending",))++ @pytest.mark.asyncio+ async def test_gives_up_at_its_statement_timeout(self, conn, conn_b, _db_url):+ await _insert_batch(conn, batch_index=0)+ # A lock that the check must wait for stands in for partition DDL on the queue DB.+ async with conn_b.transaction():+ await conn_b.execute(f"LOCK TABLE {BATCH_TABLE} IN ACCESS EXCLUSIVE MODE")+ with pytest.raises(psycopg.errors.QueryCanceled):+ BatchQueue.get_unfinished_earlier_batches_sync(+ _db_url,+ team_id=1,+ schema_id="schema-1",+ run_uuid="run-1",+ before_batch_index=3,+ connect_timeout_seconds=5,+ statement_timeout_ms=50,+ )++ # A canceled check leaves the pooled connection usable.+ assert self._check(_db_url).count == 1++@pytest.mark.django_db(transaction=True)class TestClaimWindowSkipsForeignLeasedGroups:@pytest.mark.asynciodiff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/queue_replay.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/queue_replay.pyindex 17ff427cdd1f..8bfe759e1428 100644--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/queue_replay.py+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/queue_replay.py@@ -22,7 +22,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipelineprocess_message,)from products.warehouse_sources.backend.types import ExternalDataJobStatus-from products.warehouse_sources_queue.backend.core.jobs_db import BATCH_TABLE, PendingBatch+from products.warehouse_sources_queue.backend.core.jobs_db import BATCH_TABLE, PendingBatch, build_status_dual_write_sqlfrom products.warehouse_sources_queue.backend.testing import ensure_queue_tables, get_test_database_urlPRODUCER_DATABASE_URL_PATH = (@@ -69,7 +69,15 @@ class PostgresQueueReplay:try:process_message(batch.to_export_signal())except Exception:- pass+ continue+ # The real consumer records each loaded batch, and a final batch completes its job only+ # when every earlier batch of the run has this state. The write must commit outside the+ # test transaction, because the loader reads the queue on its own connection.+ with psycopg.connect(get_test_database_url(), autocommit=True) as queue_conn:+ queue_conn.execute(+ build_status_dual_write_sql(with_batch_created_at=False),+ {"batch_id": row["id"], "job_state": "succeeded", "attempt": 1, "error_response": None},+ )def get_run_uuids_for_job(self, job_id: str) -> list[str]:with django_conn.cursor() as cur:@@ -155,6 +163,7 @@ async def replay_v3_consumer(DATA_WAREHOUSE_REDIS_HOST="localhost",DATA_WAREHOUSE_REDIS_PORT="6379",DATAWAREHOUSE_BUCKET=bucket_name,+ WAREHOUSE_SOURCES_DATABASE_URL=get_test_database_url(),),mock.patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.load.processor.is_batch_already_processed",diff --git a/products/warehouse_sources_queue/backend/README.md b/products/warehouse_sources_queue/backend/README.mdindex c73d2418c771..dfdc14ab4d7c 100644--- a/products/warehouse_sources_queue/backend/README.md+++ b/products/warehouse_sources_queue/backend/README.md@@ -64,6 +64,9 @@ All SQL lives in `core/jobs_db.py`; the polling/retry/recovery engine is `core/bIt is single-flighted fleet-wide through a sentinel lease row (`try_acquire_reconcile_sweep_slot`, team_id 0 / `__reconcile-sweep__`, 240s slot TTL): the sweeps reconcile global state, so N pods running them concurrently is pure duplicated load, exactly when the queue DB can least afford it.- **Stranded-run sweep** (same cadence and connection, `get_stale_stranded_runs`): runs the loader abandoned, meaning non-terminal batches, no live lease, and no loader progress for 6 hours.These have no `failed` batch for the reconcile sweep to key on (the extraction died before a final batch), so without this pass they would strand until the retention prune.+ - **Aged-out run sweep** (same cadence and connection, `get_aged_out_runs`): runs that hold a `pending` or `waiting_retry` batch older than `CLAIM_ELIGIBILITY_INTERVAL`.+ No claim offers that batch again, but the run gates do not read `pending` rows, so the later batches of the run stay claimable. The sweep fails the run's batches, then the job, then releases the pipeline lock.+ The stranded-run sweep does not catch these runs, because the group makes progress on the later batches.- **Sync producer** (`pipeline_v3/postgres_queue/producer.py`, in warehouse_sources): runs inside Temporal activities, plain `psycopg.Connection` with autocommit. Each queue row is a single INSERT.The v3 pipeline holds each batch's row back by one (`hold_batch`): staging batch N inserts the row for batch N-1, and the end of the run inserts the held row with `is_final_batch = true`, so the run's last data row is its own final marker and the loader reads every parquet file once.A resumable source releases the held row before it commits a cursor (`release_held_batch`), because the cursor promises that every row before it is loadable; when that leaves nothing held at the end, `send_final_batch` inserts the last batch a second time as a final-only marker, which the loader still accepts (`_run_post_load_for_already_processed_batch`).@@ -82,6 +85,7 @@ All SQL lives in `core/jobs_db.py`; the polling/retry/recovery engine is `core/bThe bucket's 8-day lifecycle rule for extraction files also stays above this window, so a claimable batch's parquet still exists.`TestClaimEligibilityWindow` in `products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/postgres_queue/test_jobs_db.py` enforces the retention coupling.The bucket rule lives in the infrastructure repo, and nothing in this repo checks it.+ As a second guard, the loader refuses to complete a job when an earlier batch of the final batch's run did not reach `succeeded` (`_refuse_incomplete_run` in `load/processor.py`). The consumer then fails the run, so the staged cursor never moves past rows that did not load.- **Partition pruning bounds**: consumer queries include `created_at > now() - interval '14 days'` (2x the retention) so the planner can skip dropped partitions.### Query cost scales with the answer, not with history@@ -129,7 +133,7 @@ The overall topology (Temporal producer activity, two tables, a fleet of consumeThe consumer engine emits nine `warehouse_pg_consumer_*` metrics (`metrics.py`): `batches_processed_total`, `batch_processing_duration_seconds`, `batch_retry_total`, `runs_failed_total`, `poll_duration_seconds`, `poll_batches_fetched`, `poll_failures_total`, `active_groups`, `recovery_sweeps_total`.Other sinks get that nine-metric engine set under their own `{prefix}_pg_consumer_*` names via `make_consumer_metrics`, so dashboards and KEDA queries never conflate two consumers' series.-Two further counters exist only under the `warehouse_pg_consumer_*` prefix, incremented by the Delta consumer's reconcile sweep (`consumer.py`) rather than by the shared engine: `runs_reconciled_total` and `runs_terminalized_stale_total`.+Further counters exist only under the `warehouse_pg_consumer_*` prefix, incremented by the Delta consumer's reconcile sweep (`consumer.py`) rather than by the shared engine: `runs_reconciled_total`, `runs_terminalized_stale_total` and `runs_failed_aged_out_total`.The headline health signal is `warehouse_pg_queue_oldest_unclaimed_batch_seconds`: the age of the oldest batch no consumer has picked up yet.It is the loader's data-freshness signal and rises whenever loading stalls, regardless of cause, so its alert fires even when every other signal looks green.diff --git a/products/warehouse_sources_queue/backend/core/jobs_db.py b/products/warehouse_sources_queue/backend/core/jobs_db.pyindex 5dd855cfa2bf..f693017e6de9 100644--- a/products/warehouse_sources_queue/backend/core/jobs_db.py+++ b/products/warehouse_sources_queue/backend/core/jobs_db.py@@ -99,7 +99,8 @@ PARTITION_PRUNING_INTERVAL = "14 days"# batch's file still exists. Applies only to the outer claim# candidates and the recovery sweep — never to the head-of-line/failed-run/# schema-busy gates, which must keep seeing rows aged past this window for as-# long as they exist.+# long as they exist. A 'pending' or 'waiting_retry' batch past this window can+# never load, so the reconcile sweep fails its run (see `get_aged_out_runs`).CLAIM_ELIGIBILITY_INTERVAL = "6 days 12 hours"# One claim reads a window of the queue, not the whole claimable set, so its cost@@ -935,6 +936,71 @@ def _stranded_candidate_runs_sql() -> str:"""+def _aged_out_candidate_runs_sql() -> str:+ """Runs that hold a 'pending' or 'waiting_retry' batch older than ``CLAIM_ELIGIBILITY_INTERVAL``.++ The claim query never offers such a batch again, and the run gates do not+ read 'pending' rows. So the later batches of the run stay claimable, and+ the run can load past the batch that will never load. The reconcile sweep+ fails these runs before their final batch can complete the job.++ The candidate states are exactly ``sb_claimable_idx``'s, for the same reason+ as in :func:`_orphaned_candidate_runs_sql`. Runs that hold a 'failed' batch+ are left to the failed-run reconcile and the orphan drain. The ``OFFSET 0``+ fence keeps that gate one ``sb_run_gate_idx`` probe per candidate run.+ """+ return f"""+ WITH aged_runs AS (+ SELECT+ b.run_uuid,+ b.team_id,+ b.schema_id,+ MAX(b.job_id) AS job_id,+ MAX(b.metadata->>'workflow_run_id') AS workflow_run_id,+ MIN(b.created_at) AS oldest_created_at,+ MIN(b.batch_index) AS first_aged_out_batch_index,+ count(*) AS aged_out_batches+ FROM {BATCH_TABLE} b+ WHERE b.created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'+ AND b.created_at <= now() - interval '{CLAIM_ELIGIBILITY_INTERVAL}'+ AND b.latest_state IN ('pending', 'waiting_retry')+ GROUP BY b.run_uuid, b.team_id, b.schema_id+ )+ SELECT+ r.run_uuid, r.team_id, r.schema_id, r.job_id, r.workflow_run_id,+ r.first_aged_out_batch_index, r.aged_out_batches+ FROM aged_runs r+ WHERE NOT EXISTS (+ SELECT 1+ FROM {BATCH_TABLE} bf+ WHERE bf.run_uuid = r.run_uuid+ AND bf.created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'+ AND bf.latest_state = 'failed'+ OFFSET 0+ )+ ORDER BY r.oldest_created_at ASC+ LIMIT %(limit)s+ """+++# Server-side ceiling for the completion check of a final batch. A lock wait (for example behind+# partition DDL) then fails the attempt, and the batch retries, instead of holding the loader thread.+UNFINISHED_EARLIER_BATCHES_STATEMENT_TIMEOUT_MS = 30_000++# Reads the earlier batches of one run through ``sb_run_uuid_bi_idx``. The team and+# schema filters guard against a run_uuid collision across groups.+_UNFINISHED_EARLIER_BATCHES_SQL = f"""+ SELECT count(*), MIN(b.batch_index), COALESCE(array_agg(DISTINCT b.latest_state), '{{}}')+ FROM {BATCH_TABLE} b+ WHERE b.run_uuid = %(run_uuid)s+ AND b.batch_index < %(before_batch_index)s+ AND b.team_id = %(team_id)s+ AND b.schema_id = %(schema_id)s+ AND b.created_at > now() - interval '{PARTITION_PRUNING_INTERVAL}'+ AND b.latest_state <> 'succeeded'+"""++def _stale_executing_sql(scope_sql: str = "") -> str:"""Shared body of the stale-executing sweep (async consumer and its sync ops twin).@@ -1159,6 +1225,33 @@ class StrandedRunRef:non_terminal_batches: int+@dataclass(frozen=True, slots=True)+class AgedOutRunRef:+ """A run with a batch that aged out of ``CLAIM_ELIGIBILITY_INTERVAL`` before it loaded.++ The batch will never load, because no claim offers it again and its parquet+ goes with the bucket lifecycle rule. The run can therefore never complete+ correctly, and the reconcile sweep fails it.+ """++ run_uuid: str+ job_id: str+ team_id: int+ schema_id: str+ workflow_run_id: str | None+ first_aged_out_batch_index: int+ aged_out_batches: int+++@dataclass(frozen=True, slots=True)+class UnfinishedEarlierBatches:+ """The batches of a run, before a given batch_index, whose latest state is not 'succeeded'."""++ count: int+ first_batch_index: int | None+ states: tuple[str, ...]++@dataclass(frozen=True, slots=True)class OrphanedRunRef:"""A run holding a ``failed`` batch that still has non-terminal batches behind it.@@ -1957,6 +2050,55 @@ class BatchQueue:raiseraise AssertionError("unreachable")+ @staticmethod+ def get_unfinished_earlier_batches_sync(+ database_url: str,+ *,+ team_id: int,+ schema_id: str,+ run_uuid: str,+ before_batch_index: int,+ connect_timeout_seconds: int = 10,+ statement_timeout_ms: int = UNFINISHED_EARLIER_BATCHES_STATEMENT_TIMEOUT_MS,+ ) -> UnfinishedEarlierBatches:+ """The batches of a run before ``before_batch_index`` that did not reach 'succeeded'.++ The loader asks this before a final batch completes its job. A batch that+ a run skipped (aged out, failed, still queued) means that the run is not+ complete, so the staged cursor must not advance past it. Runs on the same+ small sync pool as :meth:`verify_group_lease_sync`, with the same retry for a+ connection the queue DB dropped while idle.++ Raises ``psycopg.errors.QueryCanceled`` past ``statement_timeout_ms``.+ """+ pool = _sync_connection_pool(database_url, connect_timeout_seconds)+ params = {+ "team_id": team_id,+ "schema_id": schema_id,+ "run_uuid": run_uuid,+ "before_batch_index": before_batch_index,+ }+ for attempt in (1, 2):+ with pool.connection() as conn:+ try:+ # The timeout is transaction-local, so it never leaks into the pooled session.+ with conn.transaction():+ conn.execute("SELECT set_config('statement_timeout', %s, true)", (str(statement_timeout_ms),))+ row = conn.execute(_UNFINISHED_EARLIER_BATCHES_SQL, params).fetchone()+ except psycopg.errors.QueryCanceled:+ raise+ except psycopg.OperationalError:+ pool.discard(conn)+ if attempt == 2:+ raise+ continue+ assert row is not None+ count, first_batch_index, states = row+ return UnfinishedEarlierBatches(+ count=count, first_batch_index=first_batch_index, states=tuple(sorted(states))+ )+ raise AssertionError("unreachable")+@staticmethodasync def get_stale_executing(conn: psycopg.AsyncConnection[Any],@@ -2358,6 +2500,35 @@ class BatchQueue:for row in rows]+ @staticmethod+ async def get_aged_out_runs(+ conn: psycopg.AsyncConnection[Any],+ *,+ limit: int,+ ) -> list[AgedOutRunRef]:+ """Runs with a non-terminal batch past ``CLAIM_ELIGIBILITY_INTERVAL``, oldest first.++ The stranded-run sweep cannot catch these. It spares a group that makes+ progress, and a long serialized run makes progress on its later batches+ while an earlier one ages out. See :func:`_aged_out_candidate_runs_sql`.+ """+ async with conn.cursor(row_factory=dict_row) as cur:+ await cur.execute(_aged_out_candidate_runs_sql(), {"limit": limit})+ rows = await cur.fetchall()++ return [+ AgedOutRunRef(+ run_uuid=row["run_uuid"],+ job_id=row["job_id"],+ team_id=row["team_id"],+ schema_id=row["schema_id"],+ workflow_run_id=row["workflow_run_id"],+ first_aged_out_batch_index=row["first_aged_out_batch_index"],+ aged_out_batches=row["aged_out_batches"],+ )+ for row in rows+ ]+@staticmethodasync def get_queue_freshness(conn: psycopg.AsyncConnection[Any],diff --git a/products/warehouse_sources_queue/backend/core/metrics.py b/products/warehouse_sources_queue/backend/core/metrics.pyindex 503e2067c660..05bdcaca3087 100644--- a/products/warehouse_sources_queue/backend/core/metrics.py+++ b/products/warehouse_sources_queue/backend/core/metrics.py@@ -107,6 +107,13 @@ RUNS_TERMINALIZED_STALE_TOTAL = Counter("staleness threshold) that the reconcile sweep failed before the retention prune would",)+RUNS_FAILED_AGED_OUT_TOTAL = Counter(+ "warehouse_pg_consumer_runs_failed_aged_out_total",+ "Runs the reconcile sweep failed because a batch aged out of the claim window before it "+ "loaded. Without the sweep, the later batches of such a run could complete the job past "+ "the batch that never loaded",+)+# The loader's data-freshness signal: it rises whenever loading stalls,# regardless of why (wedged consumers, claim-query degradation, crashloops).# One elected pod per fleet samples it (see clear_queue_sample_gauges), so