Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): label data_import_finished failures by streak (#125327)

estefaniac4c1fae9a61d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
index b94ab2779dba..27d2a84c4d58 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
@@ -60,6 +60,7 @@ from products.warehouse_sources.backend.temporal.data_imports.external_product_h
PersonPropertySyncActivityInputs,
)
from products.warehouse_sources.backend.temporal.data_imports.metrics import (
+ failure_streak_label,
get_data_import_finished_metric,
get_fast_returned_run_metric,
get_import_handoffs_per_run_metric,
@@ -972,6 +973,8 @@ class ExternalDataJobWorkflow(PostHogWorkflow):
)
source_type = None
+ # The finally block reads it, so it must exist when job creation itself fails.
+ failed_runs_in_a_row = 0
consumer_manages_job_status = False
skip_post_import_activities = False
workflow_starts_post_import = False
@@ -1086,7 +1089,6 @@ class ExternalDataJobWorkflow(PostHogWorkflow):
hit_billing_limit = False
source_templates_needed = True
import_handoffs_are_free = False
- failed_runs_in_a_row = 0
else:
job_id = create_job_result.job_id
incremental_or_append = create_job_result.incremental_or_append
@@ -1513,7 +1515,12 @@ class ExternalDataJobWorkflow(PostHogWorkflow):
consumer_manages_job_status and update_inputs.status == ExternalDataJob.Status.COMPLETED
)
- get_data_import_finished_metric(source_type=source_type, status=update_inputs.status.lower()).add(1)
+ finished_status = update_inputs.status.lower()
+ get_data_import_finished_metric(
+ source_type=source_type,
+ status=finished_status,
+ streak=failure_streak_label(finished_status, failed_runs_in_a_row),
+ ).add(1)
# This run still owns the v3 lock when no batch reached the loader (extraction failed
# before producing batches, or a zero-batch run); the loader releases it otherwise.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/metrics.py
index ce563a64dd11..609cd119bec3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/metrics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/metrics.py
@@ -1,6 +1,6 @@
import logging
import datetime as dt
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
from temporalio import activity, workflow
from temporalio.common import MetricCounter, MetricHistogram, MetricHistogramFloat
@@ -10,6 +10,8 @@ from posthog.kafka_client.topics import KAFKA_APP_METRICS2
from posthog.models.event.util import format_clickhouse_timestamp
from products.warehouse_sources.backend.models.external_data_destination import get_or_create_warehouse_destination
+from products.warehouse_sources.backend.temporal.data_imports.retry_limits import GIVE_UP_AFTER_FAILED_RUNS
+from products.warehouse_sources.backend.types import ExternalDataJobStatus
if TYPE_CHECKING:
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
@@ -41,11 +43,28 @@ LOCK_TAKEOVER_LATEST_ERROR = (
)
-def get_data_import_finished_metric(source_type: str | None, status: str) -> MetricCounter:
+FailureStreak = Literal["new", "ongoing", "none"]
+
+
+def failure_streak_label(status: str, failed_runs_in_a_row: int) -> FailureStreak:
+ """`streak` label for `data_import_finished`. `failed_runs_in_a_row` is the streak before this run.
+
+ A failure past the give-up threshold is "ongoing", usually a broken customer connection, so
+ alerting on "new" failures counts only sources that just broke. Non-failed runs get "none" so
+ every series carries the same label names.
+ """
+ if status.lower() != ExternalDataJobStatus.FAILED.lower():
+ return "none"
+ if failed_runs_in_a_row < GIVE_UP_AFTER_FAILED_RUNS:
+ return "new"
+ return "ongoing"
+
+
+def get_data_import_finished_metric(source_type: str | None, status: str, streak: FailureStreak) -> MetricCounter:
source_type = source_type or "unknown"
return (
workflow.metric_meter()
- .with_additional_attributes({"source_type": source_type, "status": status})
+ .with_additional_attributes({"source_type": source_type, "status": status, "streak": streak})
.create_counter("data_import_finished", "Number of data imports finished, for any reason (including failure).")
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/conftest.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/conftest.py
index d2ea5581f063..01d3390bb2c9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/conftest.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/conftest.py
@@ -234,7 +234,9 @@ async def run_external_data_job_workflow(
mock_run_scheduled.assert_called()
mock_get_data_import_finished_metric.assert_called_with(
- source_type=external_data_source.source_type, status=ExternalDataJob.Status.COMPLETED.lower()
+ source_type=external_data_source.source_type,
+ status=ExternalDataJob.Status.COMPLETED.lower(),
+ streak="none",
)
await external_data_schema.arefresh_from_db()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py
index d1c0cdc988e4..61b8db733a74 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py
@@ -418,7 +418,9 @@ async def _run(
else:
mock_run_scheduled.assert_called()
mock_get_data_import_finished_metric.assert_called_with(
- source_type=source_type, status=ExternalDataJobStatus.COMPLETED.lower()
+ source_type=source_type,
+ status=ExternalDataJobStatus.COMPLETED.lower(),
+ streak="none",
)
# Assert that app_metrics2 rows were emitted for the successful job — both
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_retry_limits.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_retry_limits.py
index 0a0bea34e702..90d330e57d34 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_retry_limits.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_retry_limits.py
@@ -4,6 +4,8 @@ from django.test import SimpleTestCase
from parameterized import parameterized
+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
+from products.warehouse_sources.backend.temporal.data_imports.metrics import failure_streak_label
from products.warehouse_sources.backend.temporal.data_imports.retry_limits import (
FAILING_SCHEMA_MAX_IMPORT_ATTEMPTS,
FAILING_SCHEMA_RUN_GAP_BASE,
@@ -49,3 +51,37 @@ class TestMinGapBetweenRuns(SimpleTestCase):
gap = min_gap_between_runs(failed_runs)
assert gap is not None
assert gap <= FAILING_SCHEMA_RUN_GAP_CAP
+
+
+class TestFailureStreakLabel(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("first_failure_is_new", ExternalDataJob.Status.FAILED, 0, "new"),
+ (
+ "last_failure_before_the_threshold_is_new",
+ ExternalDataJob.Status.FAILED,
+ GIVE_UP_AFTER_FAILED_RUNS - 1,
+ "new",
+ ),
+ (
+ "failure_at_the_threshold_is_ongoing",
+ ExternalDataJob.Status.FAILED,
+ GIVE_UP_AFTER_FAILED_RUNS,
+ "ongoing",
+ ),
+ (
+ "lowercased_status_matches_too",
+ ExternalDataJob.Status.FAILED.lower(),
+ GIVE_UP_AFTER_FAILED_RUNS,
+ "ongoing",
+ ),
+ (
+ "completed_run_after_a_streak_is_none",
+ ExternalDataJob.Status.COMPLETED,
+ GIVE_UP_AFTER_FAILED_RUNS,
+ "none",
+ ),
+ ]
+ )
+ def test_label(self, _name: str, status: str, failed_runs_in_a_row: int, expected: str) -> None:
+ assert failure_streak_label(status, failed_runs_in_a_row) == expected