Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse): read a full refresh to an external destination from the start (#123746)

posthog[bot]1d99c48c0683
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py
index ac99c00e01fe..1005d9b68efa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/resumable.py
@@ -269,6 +269,16 @@ class ResumableSourceManager(Generic[ResumableData]):
self._logger.debug(f"Clearing resumable source state. key={self._key}")
self._write_with_stale_replica_retry(redis_client, lambda: redis_client.delete(self._key))
+ def clear_job_state(self) -> None:
+ """Drop the resume state of this job in every namespace, so the next read starts from the first row."""
+ base = self._job_key(self._inputs.job_id)
+ self._pending.clear()
+ self._staged.clear()
+ with self._write_fence.writing(), self._get_redis() as redis_client:
+ keys = [base, *redis_client.scan_iter(match=f"{base}:*")]
+ self._logger.debug(f"Clearing the resumable source state of the job. key={base}, keys={len(keys)}")
+ self._write_with_stale_replica_retry(redis_client, lambda: redis_client.delete(*keys))
+
def continue_from_job(self, job_id: str) -> bool:
"""Copy the resume state that `job_id` committed to this job, and return whether `job_id` had any.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py
index cb72b8796367..4a9b9c1cd3de 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_resumable.py
@@ -246,6 +246,18 @@ class TestContinueFromJob:
assert next_job.with_namespace("refunds").load_state() is None
assert next_job.load_state() is None
+ def test_clear_job_state_drops_every_namespace_of_the_job_only(self):
+ job = _manager("job-1")
+ _commit(job, "ch_100")
+ _commit(job.with_namespace("refunds"), "re_7")
+ _commit(_manager("job-2"), "ch_9")
+
+ job.clear_job_state()
+
+ assert job.load_state() is None
+ assert job.with_namespace("refunds").load_state() is None
+ assert _manager("job-2").load_state() == _SweepPosition(cursor="ch_9")
+
class TestResumeCoversRun:
@staticmethod
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
index 5e12de6bf350..9c2403db7e12 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
@@ -131,7 +131,7 @@ from products.warehouse_sources.backend.temporal.data_imports.util import (
PostHogInternalDatabaseError,
)
from products.warehouse_sources.backend.temporal.data_imports.workload_report import aworkload_reporting
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.backend.types import ExternalDataSchemaSyncType, ExternalDataSourceType
LOGGER = get_logger(__name__)
@@ -273,6 +273,20 @@ def _jobs_to_continue(job: ExternalDataJob, schema: ExternalDataSchema) -> list[
return chain
+def _publishes_staged_full_refresh(
+ job: ExternalDataJob, sync_type: ExternalDataSchemaSyncType | None, team_id: int
+) -> bool:
+ """Whether an external destination stages this full refresh and publishes it on the final batch.
+
+ A destination writer stages a full refresh under the run id of one attempt, and each attempt
+ has its own run id. An attempt that resumes from the cursor of an earlier attempt or job
+ stages only the rows after that cursor, and its final batch publishes that part as the whole table.
+ """
+ if sync_type not in (None, ExternalDataSchema.SyncType.FULL_REFRESH):
+ return False
+ return bool(external_destination_ids_for(team_id, list(job.destination_ids or [])))
+
+
@database_sync_to_async_pool
def _carry_rows_synced(job: ExternalDataJob, continued: ExternalDataJob) -> None:
"""Start the row count of `job` from the count of the job it continues.
@@ -855,7 +869,19 @@ async def _import_data_with_reporting(
try:
if isinstance(new_source, ResumableSource):
resumable_source_manager = new_source.get_resumable_source_manager(source_inputs)
- jobs_to_continue = await _jobs_to_continue(model, schema) if use_stored_cursors else []
+ reads_from_start = await database_sync_to_async_pool(_publishes_staged_full_refresh)(
+ model, resolved_sync_type, inputs.team_id
+ )
+ if reads_from_start:
+ await database_sync_to_async_pool(resumable_source_manager.clear_job_state)()
+ await logger.ainfo(
+ "Reading the source from the start: an external destination publishes this full refresh from one attempt"
+ )
+ jobs_to_continue = (
+ await _jobs_to_continue(model, schema)
+ if use_stored_cursors and not reads_from_start
+ else []
+ )
if jobs_to_continue:
job_to_continue = jobs_to_continue[0]
continued_workflow_run_id = job_to_continue.workflow_run_id
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
index e7f2a716f069..7c081010dbde 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
@@ -1364,6 +1364,37 @@ async def test_a_resumable_run_continues_the_cursor_of_the_interrupted_job_unles
assert run_mock.call_args.kwargs["continued_job_ids"] == (("interrupted-job", "earlier-job") if continues else ())
+ "is_incremental,external_destination_ids,reads_from_start",
+ [(False, ["dest-1"], True), (False, [], False), (True, ["dest-1"], False)],
+ ids=["full_refresh_to_destination", "full_refresh_to_warehouse_only", "incremental_to_destination"],
+)
+async def test_a_full_refresh_to_an_external_destination_reads_the_source_from_the_start(
+ is_incremental: bool, external_destination_ids: list[str], reads_from_start: bool
+) -> None:
+ source = mock.MagicMock(spec=ResumableSource)
+ source.parse_config.return_value = {}
+ source.source_for_pipeline.return_value = mock.MagicMock()
+ manager = source.get_resumable_source_manager.return_value
+ manager.continue_from_job.return_value = True
+ schema = _incremental_schema(is_incremental=is_incremental, lookback_seconds=None)
+ interrupted = mock.MagicMock(id="interrupted-job", workflow_run_id="interrupted-run")
+
+ with (
+ _patched_activity_reaching_run(source, schema) as run_mock,
+ mock.patch.object(module, "external_destination_ids_for", return_value=external_destination_ids),
+ mock.patch.object(module, "_jobs_to_continue", new=mock.AsyncMock(return_value=[interrupted])),
+ mock.patch.object(module, "_carry_rows_synced", new=mock.AsyncMock()),
+ ):
+ await import_data_activity_sync(_inputs_no_reset())
+
+ # The destination publishes the staging table of this attempt alone, so it must hold every row.
+ assert manager.clear_job_state.called is reads_from_start
+ assert manager.continue_from_job.called is not reads_from_start
+ assert run_mock.call_args.kwargs["continued_job_ids"] == (() if reads_from_start else ("interrupted-job",))
+
+
@pytest.mark.asyncio
async def test_pending_delta_revive_extracts_full_table_not_incremental_slice():
source = mock.MagicMock(spec=SimpleSource)