:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): label data_import_finished failures by streak (#125327)
estefaniac4c1fae9a61d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.pyindex b94ab2779dba..27d2a84c4d58 100644--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py@@ -60,6 +60,7 @@ from products.warehouse_sources.backend.temporal.data_imports.external_product_hPersonPropertySyncActivityInputs,)from products.warehouse_sources.backend.temporal.data_imports.metrics import (+ failure_streak_label,get_data_import_finished_metric,get_fast_returned_run_metric,get_import_handoffs_per_run_metric,@@ -972,6 +973,8 @@ class ExternalDataJobWorkflow(PostHogWorkflow):)source_type = None+ # The finally block reads it, so it must exist when job creation itself fails.+ failed_runs_in_a_row = 0consumer_manages_job_status = Falseskip_post_import_activities = Falseworkflow_starts_post_import = False@@ -1086,7 +1089,6 @@ class ExternalDataJobWorkflow(PostHogWorkflow):hit_billing_limit = Falsesource_templates_needed = Trueimport_handoffs_are_free = False- failed_runs_in_a_row = 0else:job_id = create_job_result.job_idincremental_or_append = create_job_result.incremental_or_append@@ -1513,7 +1515,12 @@ class ExternalDataJobWorkflow(PostHogWorkflow):consumer_manages_job_status and update_inputs.status == ExternalDataJob.Status.COMPLETED)- get_data_import_finished_metric(source_type=source_type, status=update_inputs.status.lower()).add(1)+ finished_status = update_inputs.status.lower()+ get_data_import_finished_metric(+ source_type=source_type,+ status=finished_status,+ streak=failure_streak_label(finished_status, failed_runs_in_a_row),+ ).add(1)# This run still owns the v3 lock when no batch reached the loader (extraction failed# before producing batches, or a zero-batch run); the loader releases it otherwise.diff --git a/products/warehouse_sources/backend/temporal/data_imports/metrics.py b/products/warehouse_sources/backend/temporal/data_imports/metrics.pyindex ce563a64dd11..609cd119bec3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/metrics.py+++ b/products/warehouse_sources/backend/temporal/data_imports/metrics.py@@ -1,6 +1,6 @@import loggingimport datetime as dt-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Literalfrom temporalio import activity, workflowfrom temporalio.common import MetricCounter, MetricHistogram, MetricHistogramFloat@@ -10,6 +10,8 @@ from posthog.kafka_client.topics import KAFKA_APP_METRICS2from posthog.models.event.util import format_clickhouse_timestampfrom products.warehouse_sources.backend.models.external_data_destination import get_or_create_warehouse_destination+from products.warehouse_sources.backend.temporal.data_imports.retry_limits import GIVE_UP_AFTER_FAILED_RUNS+from products.warehouse_sources.backend.types import ExternalDataJobStatusif TYPE_CHECKING:from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob@@ -41,11 +43,28 @@ LOCK_TAKEOVER_LATEST_ERROR = ()-def get_data_import_finished_metric(source_type: str | None, status: str) -> MetricCounter:+FailureStreak = Literal["new", "ongoing", "none"]+++def failure_streak_label(status: str, failed_runs_in_a_row: int) -> FailureStreak:+ """`streak` label for `data_import_finished`. `failed_runs_in_a_row` is the streak before this run.++ A failure past the give-up threshold is "ongoing", usually a broken customer connection, so+ alerting on "new" failures counts only sources that just broke. Non-failed runs get "none" so+ every series carries the same label names.+ """+ if status.lower() != ExternalDataJobStatus.FAILED.lower():+ return "none"+ if failed_runs_in_a_row < GIVE_UP_AFTER_FAILED_RUNS:+ return "new"+ return "ongoing"+++def get_data_import_finished_metric(source_type: str | None, status: str, streak: FailureStreak) -> MetricCounter:source_type = source_type or "unknown"return (workflow.metric_meter()- .with_additional_attributes({"source_type": source_type, "status": status})+ .with_additional_attributes({"source_type": source_type, "status": status, "streak": streak}).create_counter("data_import_finished", "Number of data imports finished, for any reason (including failure)."))diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/conftest.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/conftest.pyindex d2ea5581f063..01d3390bb2c9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/conftest.py+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/conftest.py@@ -234,7 +234,9 @@ async def run_external_data_job_workflow(mock_run_scheduled.assert_called()mock_get_data_import_finished_metric.assert_called_with(- source_type=external_data_source.source_type, status=ExternalDataJob.Status.COMPLETED.lower()+ source_type=external_data_source.source_type,+ status=ExternalDataJob.Status.COMPLETED.lower(),+ streak="none",)await external_data_schema.arefresh_from_db()diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.pyindex d1c0cdc988e4..61b8db733a74 100644--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py@@ -418,7 +418,9 @@ async def _run(else:mock_run_scheduled.assert_called()mock_get_data_import_finished_metric.assert_called_with(- source_type=source_type, status=ExternalDataJobStatus.COMPLETED.lower()+ source_type=source_type,+ status=ExternalDataJobStatus.COMPLETED.lower(),+ streak="none",)# Assert that app_metrics2 rows were emitted for the successful job — bothdiff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_retry_limits.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_retry_limits.pyindex 0a0bea34e702..90d330e57d34 100644--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_retry_limits.py+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_retry_limits.py@@ -4,6 +4,8 @@ from django.test import SimpleTestCasefrom parameterized import parameterized+from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob+from products.warehouse_sources.backend.temporal.data_imports.metrics import failure_streak_labelfrom products.warehouse_sources.backend.temporal.data_imports.retry_limits import (FAILING_SCHEMA_MAX_IMPORT_ATTEMPTS,FAILING_SCHEMA_RUN_GAP_BASE,@@ -49,3 +51,37 @@ class TestMinGapBetweenRuns(SimpleTestCase):gap = min_gap_between_runs(failed_runs)assert gap is not Noneassert gap <= FAILING_SCHEMA_RUN_GAP_CAP+++class TestFailureStreakLabel(SimpleTestCase):+ @parameterized.expand(+ [+ ("first_failure_is_new", ExternalDataJob.Status.FAILED, 0, "new"),+ (+ "last_failure_before_the_threshold_is_new",+ ExternalDataJob.Status.FAILED,+ GIVE_UP_AFTER_FAILED_RUNS - 1,+ "new",+ ),+ (+ "failure_at_the_threshold_is_ongoing",+ ExternalDataJob.Status.FAILED,+ GIVE_UP_AFTER_FAILED_RUNS,+ "ongoing",+ ),+ (+ "lowercased_status_matches_too",+ ExternalDataJob.Status.FAILED.lower(),+ GIVE_UP_AFTER_FAILED_RUNS,+ "ongoing",+ ),+ (+ "completed_run_after_a_streak_is_none",+ ExternalDataJob.Status.COMPLETED,+ GIVE_UP_AFTER_FAILED_RUNS,+ "none",+ ),+ ]+ )+ def test_label(self, _name: str, status: str, failed_runs_in_a_row: int, expected: str) -> None:+ assert failure_streak_label(status, failed_runs_in_a_row) == expected