Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): record import attempt duration by source and outcome (#123311)

Tom Owersd03ae5110dbd
diff --git a/posthog/temporal/common/worker.py b/posthog/temporal/common/worker.py
index 55fdc62def10..71ba1ebf40b2 100644
--- a/posthog/temporal/common/worker.py
+++ b/posthog/temporal/common/worker.py
@@ -474,6 +474,20 @@ async def create_worker(
7200.0,
21600.0,
]
+ histogram_bucket_overrides["warehouse_import_attempt_duration_seconds"] = [
+ 1.0,
+ 5.0,
+ 15.0,
+ 30.0,
+ 60.0,
+ 120.0,
+ 300.0,
+ 600.0,
+ 1800.0,
+ 3600.0,
+ 7200.0,
+ 21600.0,
+ ]
if task_queue == settings.TASKS_TASK_QUEUE:
histogram_bucket_overrides |= dict(
zip(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/metrics.py
index 59940b8c0654..b5fb93238c07 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/metrics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/metrics.py
@@ -136,6 +136,20 @@ def get_import_handoffs_per_run_metric(source_type: str | None) -> MetricHistogr
)
+def get_import_attempt_duration_metric(source_type: str | None, outcome: str) -> MetricHistogramFloat:
+ # One observation per attempt of the import activity. `outcome` is `completed`, `handed_off`,
+ # `cancelled` or `failed`.
+ return (
+ activity.metric_meter()
+ .with_additional_attributes({"source_type": source_type or "unknown", "outcome": outcome})
+ .create_histogram_float(
+ "warehouse_import_attempt_duration_seconds",
+ "Seconds from the start to the exit of one attempt of the import activity.",
+ "s",
+ )
+ )
+
+
def get_import_preempted_metric(source_type: str | None, reason: str) -> MetricCounter:
# `reason` is why the run could continue on another worker (see `PreemptionDecision`).
return (
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
index 9e00df807df3..8f37245b9d67 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
@@ -1,3 +1,4 @@
+import time
import uuid
import socket
import asyncio
@@ -5,6 +6,7 @@ import datetime as dt
import functools
import dataclasses
from concurrent.futures import ThreadPoolExecutor
+from contextvars import ContextVar
from typing import TYPE_CHECKING, Any, NoReturn, Optional
from django.conf import settings
@@ -55,6 +57,7 @@ from products.warehouse_sources.backend.temporal.data_imports.import_attempt imp
)
from products.warehouse_sources.backend.temporal.data_imports.metrics import (
TERMINAL_JOB_STATUSES,
+ get_import_attempt_duration_metric,
get_progressless_stand_down_metric,
get_worker_shutdown_handoff_metric,
)
@@ -132,6 +135,9 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType
LOGGER = get_logger(__name__)
+# The source type is known only after the job row loads. The attempt wrapper reads it on exit.
+_attempt_source_type: ContextVar[str | None] = ContextVar("import_attempt_source_type", default=None)
+
@frozen
class ImportDataActivityInputs:
@@ -442,6 +448,27 @@ def v3_pipeline_class(source_response: SourceResponse) -> "type[PipelineV3]":
@activity.defn
async def import_data_activity_sync(inputs: ImportDataActivityInputs) -> PipelineResult:
+ started_at = time.monotonic()
+ outcome = "failed"
+ _attempt_source_type.set(None)
+ try:
+ result = await _run_import_attempt(inputs)
+ outcome = "handed_off" if result.get("handed_off") else "completed"
+ return result
+ except WorkerShuttingDownError:
+ outcome = "handed_off"
+ raise
+ except asyncio.CancelledError:
+ outcome = "cancelled"
+ raise
+ finally:
+ if activity.in_activity():
+ get_import_attempt_duration_metric(_attempt_source_type.get(), outcome).record(
+ time.monotonic() - started_at
+ )
+
+
+async def _run_import_attempt(inputs: ImportDataActivityInputs) -> PipelineResult:
bind_contextvars(team_id=inputs.team_id)
logger = LOGGER.bind()
tag_queries(team_id=inputs.team_id, product=Product.WAREHOUSE, feature=Feature.IMPORT_PIPELINE)
@@ -549,6 +576,7 @@ async def _import_data_with_reporting(
await logger.adebug("Running import_data_activity")
source_type = ExternalDataSourceType(model.pipeline.source_type)
+ _attempt_source_type.set(str(source_type))
bind_job_context(
team_id=inputs.team_id,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
index b475aad8d288..41a4e86cad37 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
@@ -1239,7 +1239,9 @@ def _patched_activity_reaching_run(source_mock, schema, api_version=None, workfl
mock.patch.object(module, "is_schema_flag_enabled", return_value=False),
mock.patch.object(module.SourceRegistry, "is_registered", return_value=True),
mock.patch.object(module.SourceRegistry, "get_source", return_value=source_mock),
- mock.patch.object(module, "_run", new=mock.AsyncMock(return_value=mock.sentinel.run_result)) as run_mock,
+ mock.patch.object(
+ module, "_run", new=mock.AsyncMock(return_value={"should_trigger_cdp_producer": False})
+ ) as run_mock,
):
yield run_mock
@@ -2048,6 +2050,59 @@ async def test_a_free_handoff_is_a_result_and_any_other_handoff_is_a_retry(
await import_data_activity_sync(inputs)
+ "behavior,handoffs_are_free,expected_outcome",
+ [
+ ("completes", False, "completed"),
+ ("hands_off", True, "handed_off"),
+ ("hands_off", False, "handed_off"),
+ ("raises", False, "failed"),
+ ("cancelled", False, "cancelled"),
+ ],
+)
+async def test_each_attempt_records_one_duration_with_its_outcome(
+ behavior: str, handoffs_are_free: bool, expected_outcome: str
+):
+ shutdown = WorkerShuttingDownError("5", "import_data_activity_sync", "data-warehouse-task-queue", 2, "wf", "wt")
+
+ async def import_attempt(_inputs, _logger, _progress):
+ module._attempt_source_type.set("MongoDB")
+ if behavior == "hands_off":
+ raise shutdown
+ if behavior == "raises":
+ raise ValueError("boom")
+ if behavior == "cancelled":
+ raise asyncio.CancelledError()
+ return {"should_trigger_cdp_producer": False}
+
+ inputs = dataclasses.replace(_inputs_no_reset(), handoffs_are_free=handoffs_are_free)
+
+ with (
+ mock.patch.object(module, "tag_queries"),
+ mock.patch.object(module, "report_heartbeat_timeout"),
+ mock.patch.object(module, "aworkload_reporting", return_value=_FakeAsyncCM()),
+ mock.patch.object(module, "_import_data_with_reporting", new=mock.AsyncMock(side_effect=import_attempt)),
+ mock.patch.object(module, "current_activity_attempt", return_value=1),
+ mock.patch.object(module, "get_import_attempt_duration_metric") as metric_mock,
+ mock.patch.object(module.activity, "in_activity", return_value=True),
+ ):
+ if behavior == "completes" or (behavior == "hands_off" and handoffs_are_free):
+ await import_data_activity_sync(inputs)
+ else:
+ expected_error = {
+ "hands_off": WorkerShuttingDownError,
+ "raises": ValueError,
+ "cancelled": asyncio.CancelledError,
+ }[behavior]
+ with pytest.raises(expected_error):
+ await import_data_activity_sync(inputs)
+
+ metric_mock.assert_called_once_with("MongoDB", expected_outcome)
+ metric_mock.return_value.record.assert_called_once()
+ assert metric_mock.return_value.record.call_args.args[0] >= 0
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
"preemption_enabled,handoffs_are_free,carry_over_enabled,expected",