:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse): read a full refresh to an external destination from the start (#123746)
posthog[bot]1d99c48c0683
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.pyindex ac99c00e01fe..1005d9b68efa 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py@@ -269,6 +269,16 @@ class ResumableSourceManager(Generic[ResumableData]):self._logger.debug(f"Clearing resumable source state. key={self._key}")self._write_with_stale_replica_retry(redis_client, lambda: redis_client.delete(self._key))+ def clear_job_state(self) -> None:+ """Drop the resume state of this job in every namespace, so the next read starts from the first row."""+ base = self._job_key(self._inputs.job_id)+ self._pending.clear()+ self._staged.clear()+ with self._write_fence.writing(), self._get_redis() as redis_client:+ keys = [base, *redis_client.scan_iter(match=f"{base}:*")]+ self._logger.debug(f"Clearing the resumable source state of the job. key={base}, keys={len(keys)}")+ self._write_with_stale_replica_retry(redis_client, lambda: redis_client.delete(*keys))+def continue_from_job(self, job_id: str) -> bool:"""Copy the resume state that `job_id` committed to this job, and return whether `job_id` had any.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.pyindex cb72b8796367..4a9b9c1cd3de 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py@@ -246,6 +246,18 @@ class TestContinueFromJob:assert next_job.with_namespace("refunds").load_state() is Noneassert next_job.load_state() is None+ def test_clear_job_state_drops_every_namespace_of_the_job_only(self):+ job = _manager("job-1")+ _commit(job, "ch_100")+ _commit(job.with_namespace("refunds"), "re_7")+ _commit(_manager("job-2"), "ch_9")++ job.clear_job_state()++ assert job.load_state() is None+ assert job.with_namespace("refunds").load_state() is None+ assert _manager("job-2").load_state() == _SweepPosition(cursor="ch_9")+class TestResumeCoversRun:@staticmethoddiff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.pyindex 5e12de6bf350..9c2403db7e12 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py@@ -131,7 +131,7 @@ from products.warehouse_sources.backend.temporal.data_imports.util import (PostHogInternalDatabaseError,)from products.warehouse_sources.backend.temporal.data_imports.workload_report import aworkload_reporting-from products.warehouse_sources.backend.types import ExternalDataSourceType+from products.warehouse_sources.backend.types import ExternalDataSchemaSyncType, ExternalDataSourceTypeLOGGER = get_logger(__name__)@@ -273,6 +273,20 @@ def _jobs_to_continue(job: ExternalDataJob, schema: ExternalDataSchema) -> list[return chain+def _publishes_staged_full_refresh(+ job: ExternalDataJob, sync_type: ExternalDataSchemaSyncType | None, team_id: int+) -> bool:+ """Whether an external destination stages this full refresh and publishes it on the final batch.++ A destination writer stages a full refresh under the run id of one attempt, and each attempt+ has its own run id. An attempt that resumes from the cursor of an earlier attempt or job+ stages only the rows after that cursor, and its final batch publishes that part as the whole table.+ """+ if sync_type not in (None, ExternalDataSchema.SyncType.FULL_REFRESH):+ return False+ return bool(external_destination_ids_for(team_id, list(job.destination_ids or [])))++@database_sync_to_async_pooldef _carry_rows_synced(job: ExternalDataJob, continued: ExternalDataJob) -> None:"""Start the row count of `job` from the count of the job it continues.@@ -855,7 +869,19 @@ async def _import_data_with_reporting(try:if isinstance(new_source, ResumableSource):resumable_source_manager = new_source.get_resumable_source_manager(source_inputs)- jobs_to_continue = await _jobs_to_continue(model, schema) if use_stored_cursors else []+ reads_from_start = await database_sync_to_async_pool(_publishes_staged_full_refresh)(+ model, resolved_sync_type, inputs.team_id+ )+ if reads_from_start:+ await database_sync_to_async_pool(resumable_source_manager.clear_job_state)()+ await logger.ainfo(+ "Reading the source from the start: an external destination publishes this full refresh from one attempt"+ )+ jobs_to_continue = (+ await _jobs_to_continue(model, schema)+ if use_stored_cursors and not reads_from_start+ else []+ )if jobs_to_continue:job_to_continue = jobs_to_continue[0]continued_workflow_run_id = job_to_continue.workflow_run_iddiff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.pyindex e7f2a716f069..7c081010dbde 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py@@ -1364,6 +1364,37 @@ async def test_a_resumable_run_continues_the_cursor_of_the_interrupted_job_unlesassert run_mock.call_args.kwargs["continued_job_ids"] == (("interrupted-job", "earlier-job") if continues else ())+ "is_incremental,external_destination_ids,reads_from_start",+ [(False, ["dest-1"], True), (False, [], False), (True, ["dest-1"], False)],+ ids=["full_refresh_to_destination", "full_refresh_to_warehouse_only", "incremental_to_destination"],+)+async def test_a_full_refresh_to_an_external_destination_reads_the_source_from_the_start(+ is_incremental: bool, external_destination_ids: list[str], reads_from_start: bool+) -> None:+ source = mock.MagicMock(spec=ResumableSource)+ source.parse_config.return_value = {}+ source.source_for_pipeline.return_value = mock.MagicMock()+ manager = source.get_resumable_source_manager.return_value+ manager.continue_from_job.return_value = True+ schema = _incremental_schema(is_incremental=is_incremental, lookback_seconds=None)+ interrupted = mock.MagicMock(id="interrupted-job", workflow_run_id="interrupted-run")++ with (+ _patched_activity_reaching_run(source, schema) as run_mock,+ mock.patch.object(module, "external_destination_ids_for", return_value=external_destination_ids),+ mock.patch.object(module, "_jobs_to_continue", new=mock.AsyncMock(return_value=[interrupted])),+ mock.patch.object(module, "_carry_rows_synced", new=mock.AsyncMock()),+ ):+ await import_data_activity_sync(_inputs_no_reset())++ # The destination publishes the staging table of this attempt alone, so it must hold every row.+ assert manager.clear_job_state.called is reads_from_start+ assert manager.continue_from_job.called is not reads_from_start+ assert run_mock.call_args.kwargs["continued_job_ids"] == (() if reads_from_start else ("interrupted-job",))++@pytest.mark.asyncioasync def test_pending_delta_revive_extracts_full_table_not_incremental_slice():source = mock.MagicMock(spec=SimpleSource)