:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): record import attempt duration by source and outcome (#123311)
Tom Owersd03ae5110dbd
diff --git a/posthog/temporal/common/worker.py b/posthog/temporal/common/worker.pyindex 55fdc62def10..71ba1ebf40b2 100644--- a/posthog/temporal/common/worker.py+++ b/posthog/temporal/common/worker.py@@ -474,6 +474,20 @@ async def create_worker(7200.0,21600.0,]+ histogram_bucket_overrides["warehouse_import_attempt_duration_seconds"] = [+ 1.0,+ 5.0,+ 15.0,+ 30.0,+ 60.0,+ 120.0,+ 300.0,+ 600.0,+ 1800.0,+ 3600.0,+ 7200.0,+ 21600.0,+ ]if task_queue == settings.TASKS_TASK_QUEUE:histogram_bucket_overrides |= dict(zip(diff --git a/products/warehouse_sources/backend/temporal/data_imports/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/metrics.pyindex 59940b8c0654..b5fb93238c07 100644--- a/products/warehouse_sources/backend/temporal/data_imports/metrics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/metrics.py@@ -136,6 +136,20 @@ def get_import_handoffs_per_run_metric(source_type: str | None) -> MetricHistogr)+def get_import_attempt_duration_metric(source_type: str | None, outcome: str) -> MetricHistogramFloat:+ # One observation per attempt of the import activity. `outcome` is `completed`, `handed_off`,+ # `cancelled` or `failed`.+ return (+ activity.metric_meter()+ .with_additional_attributes({"source_type": source_type or "unknown", "outcome": outcome})+ .create_histogram_float(+ "warehouse_import_attempt_duration_seconds",+ "Seconds from the start to the exit of one attempt of the import activity.",+ "s",+ )+ )++def get_import_preempted_metric(source_type: str | None, reason: str) -> MetricCounter:# `reason` is why the run could continue on another worker (see `PreemptionDecision`).return (diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.pyindex 9e00df807df3..8f37245b9d67 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py@@ -1,3 +1,4 @@+import timeimport uuidimport socketimport asyncio@@ -5,6 +6,7 @@ import datetime as dtimport functoolsimport dataclassesfrom concurrent.futures import ThreadPoolExecutor+from contextvars import ContextVarfrom typing import TYPE_CHECKING, Any, NoReturn, Optionalfrom django.conf import settings@@ -55,6 +57,7 @@ from products.warehouse_sources.backend.temporal.data_imports.import_attempt imp)from products.warehouse_sources.backend.temporal.data_imports.metrics import (TERMINAL_JOB_STATUSES,+ get_import_attempt_duration_metric,get_progressless_stand_down_metric,get_worker_shutdown_handoff_metric,)@@ -132,6 +135,9 @@ from products.warehouse_sources.backend.types import ExternalDataSourceTypeLOGGER = get_logger(__name__)+# The source type is known only after the job row loads. The attempt wrapper reads it on exit.+_attempt_source_type: ContextVar[str | None] = ContextVar("import_attempt_source_type", default=None)+@frozenclass ImportDataActivityInputs:@@ -442,6 +448,27 @@ def v3_pipeline_class(source_response: SourceResponse) -> "type[PipelineV3]":@activity.defnasync def import_data_activity_sync(inputs: ImportDataActivityInputs) -> PipelineResult:+ started_at = time.monotonic()+ outcome = "failed"+ _attempt_source_type.set(None)+ try:+ result = await _run_import_attempt(inputs)+ outcome = "handed_off" if result.get("handed_off") else "completed"+ return result+ except WorkerShuttingDownError:+ outcome = "handed_off"+ raise+ except asyncio.CancelledError:+ outcome = "cancelled"+ raise+ finally:+ if activity.in_activity():+ get_import_attempt_duration_metric(_attempt_source_type.get(), outcome).record(+ time.monotonic() - started_at+ )+++async def _run_import_attempt(inputs: ImportDataActivityInputs) -> PipelineResult:bind_contextvars(team_id=inputs.team_id)logger = LOGGER.bind()tag_queries(team_id=inputs.team_id, product=Product.WAREHOUSE, feature=Feature.IMPORT_PIPELINE)@@ -549,6 +576,7 @@ async def _import_data_with_reporting(await logger.adebug("Running import_data_activity")source_type = ExternalDataSourceType(model.pipeline.source_type)+ _attempt_source_type.set(str(source_type))bind_job_context(team_id=inputs.team_id,diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.pyindex b475aad8d288..41a4e86cad37 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py@@ -1239,7 +1239,9 @@ def _patched_activity_reaching_run(source_mock, schema, api_version=None, workflmock.patch.object(module, "is_schema_flag_enabled", return_value=False),mock.patch.object(module.SourceRegistry, "is_registered", return_value=True),mock.patch.object(module.SourceRegistry, "get_source", return_value=source_mock),- mock.patch.object(module, "_run", new=mock.AsyncMock(return_value=mock.sentinel.run_result)) as run_mock,+ mock.patch.object(+ module, "_run", new=mock.AsyncMock(return_value={"should_trigger_cdp_producer": False})+ ) as run_mock,):yield run_mock@@ -2048,6 +2050,59 @@ async def test_a_free_handoff_is_a_result_and_any_other_handoff_is_a_retry(await import_data_activity_sync(inputs)+ "behavior,handoffs_are_free,expected_outcome",+ [+ ("completes", False, "completed"),+ ("hands_off", True, "handed_off"),+ ("hands_off", False, "handed_off"),+ ("raises", False, "failed"),+ ("cancelled", False, "cancelled"),+ ],+)+async def test_each_attempt_records_one_duration_with_its_outcome(+ behavior: str, handoffs_are_free: bool, expected_outcome: str+):+ shutdown = WorkerShuttingDownError("5", "import_data_activity_sync", "data-warehouse-task-queue", 2, "wf", "wt")++ async def import_attempt(_inputs, _logger, _progress):+ module._attempt_source_type.set("MongoDB")+ if behavior == "hands_off":+ raise shutdown+ if behavior == "raises":+ raise ValueError("boom")+ if behavior == "cancelled":+ raise asyncio.CancelledError()+ return {"should_trigger_cdp_producer": False}++ inputs = dataclasses.replace(_inputs_no_reset(), handoffs_are_free=handoffs_are_free)++ with (+ mock.patch.object(module, "tag_queries"),+ mock.patch.object(module, "report_heartbeat_timeout"),+ mock.patch.object(module, "aworkload_reporting", return_value=_FakeAsyncCM()),+ mock.patch.object(module, "_import_data_with_reporting", new=mock.AsyncMock(side_effect=import_attempt)),+ mock.patch.object(module, "current_activity_attempt", return_value=1),+ mock.patch.object(module, "get_import_attempt_duration_metric") as metric_mock,+ mock.patch.object(module.activity, "in_activity", return_value=True),+ ):+ if behavior == "completes" or (behavior == "hands_off" and handoffs_are_free):+ await import_data_activity_sync(inputs)+ else:+ expected_error = {+ "hands_off": WorkerShuttingDownError,+ "raises": ValueError,+ "cancelled": asyncio.CancelledError,+ }[behavior]+ with pytest.raises(expected_error):+ await import_data_activity_sync(inputs)++ metric_mock.assert_called_once_with("MongoDB", expected_outcome)+ metric_mock.return_value.record.assert_called_once()+ assert metric_mock.return_value.record.call_args.args[0] >= 0++@pytest.mark.asyncio@pytest.mark.parametrize("preemption_enabled,handoffs_are_free,carry_over_enabled,expected",