:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): clamp negative schedule-to-start latency in temporal interceptors (#122478)
posthog[bot]f201bfad0927
diff --git a/posthog/temporal/ai_observability/eval_reports/metrics.py b/posthog/temporal/ai_observability/eval_reports/metrics.pyindex ff1439698ad2..5488bb6b860e 100644--- a/posthog/temporal/ai_observability/eval_reports/metrics.py+++ b/posthog/temporal/ai_observability/eval_reports/metrics.py@@ -6,7 +6,6 @@ and scraped by the Prometheus endpoint on the worker pod."""import typing-import datetime as dtfrom django.conf import settings@@ -21,6 +20,7 @@ from temporalio.worker import ()from posthog.temporal.ai_observability.metrics import ExecutionTimeRecorder, get_metric_meter+from posthog.temporal.common.metrics import record_schedule_to_start_latency# ---------------------------------------------------------------------------# Histogram bucket config (imported by common/worker.py for PrometheusConfig)@@ -175,16 +175,13 @@ class _EvalReportsActivityInterceptor(ActivityInboundInterceptor):return await super().execute_activity(input)# Schedule-to-start latency (queue depth indicator)- scheduled_time = activity_info.scheduled_time- started_time = activity_info.started_time- if scheduled_time and started_time:- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)- meter = get_metric_meter({"activity_type": activity_type})- meter.create_histogram_timedelta(- name="llma_eval_reports_activity_schedule_to_start_latency",- description="Time between activity scheduling and start",- unit="ms",- ).record(dt.timedelta(milliseconds=schedule_to_start_ms))+ record_schedule_to_start_latency(+ "llma_eval_reports_activity_schedule_to_start_latency",+ scheduled_time=activity_info.scheduled_time,+ started_time=activity_info.started_time,+ description="Time between activity scheduling and start",+ attributes={"activity_type": activity_type},+ )with ExecutionTimeRecorder("llma_eval_reports_activity_execution_latency",diff --git a/posthog/temporal/ai_observability/metrics.py b/posthog/temporal/ai_observability/metrics.pyindex 7ee64cc752d7..98c1b97fa19a 100644--- a/posthog/temporal/ai_observability/metrics.py+++ b/posthog/temporal/ai_observability/metrics.py@@ -1,5 +1,4 @@import typing-import datetime as dtfrom django.conf import settings@@ -17,7 +16,12 @@ from posthog.temporal.common.logger import get_write_only_logger# Generic temporal metric helpers live in common so other subsystems (e.g. session-replay# delete-recordings) can import them without pulling this package's eager worker registration.-from posthog.temporal.common.metrics import Attributes, ExecutionTimeRecorder, get_metric_meter+from posthog.temporal.common.metrics import (+ Attributes,+ ExecutionTimeRecorder,+ get_metric_meter,+ record_schedule_to_start_latency,+)__all__ = ["Attributes", "ExecutionTimeRecorder", "get_metric_meter"]@@ -175,17 +179,6 @@ def increment_backfill_remainder_outcome(outcome: str) -> None:counter.add(1)-def record_schedule_to_start_latency(activity_type: str, latency_ms: int) -> None:- """Record queue depth indicator for alerting."""- meter = get_metric_meter({"activity_type": activity_type})- hist = meter.create_histogram_timedelta(- name="llma_eval_activity_schedule_to_start_latency",- description="Time between activity scheduling and start (queue depth indicator)",- unit="ms",- )- hist.record(dt.timedelta(milliseconds=latency_ms))--class EvalsMetricsInterceptor(Interceptor):"""Interceptor to emit Prometheus metrics for evals workflows."""@@ -209,11 +202,13 @@ class _EvalsMetricsActivityInboundInterceptor(ActivityInboundInterceptor):return await super().execute_activity(input)# Record schedule-to-start latency (queue depth indicator)- scheduled_time = activity_info.scheduled_time- started_time = activity_info.started_time- if scheduled_time and started_time:- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)- record_schedule_to_start_latency(activity_type, schedule_to_start_ms)+ record_schedule_to_start_latency(+ "llma_eval_activity_schedule_to_start_latency",+ scheduled_time=activity_info.scheduled_time,+ started_time=activity_info.started_time,+ description="Time between activity scheduling and start (queue depth indicator)",+ attributes={"activity_type": activity_type},+ )histogram_attributes: Attributes = {"activity_type": activity_type}diff --git a/posthog/temporal/ai_observability/trace_clustering/metrics.py b/posthog/temporal/ai_observability/trace_clustering/metrics.pyindex b190d4cfcbad..52a9b2c508d0 100644--- a/posthog/temporal/ai_observability/trace_clustering/metrics.py+++ b/posthog/temporal/ai_observability/trace_clustering/metrics.py@@ -6,7 +6,6 @@ and scraped by the Prometheus endpoint on the worker pod."""import typing-import datetime as dtfrom django.conf import settings@@ -24,6 +23,7 @@ from posthog.temporal.ai_observability.evaluation_clustering.constants import (CLUSTERING_WORKFLOW_NAME as EVAL_CLUSTERING_WORKFLOW_NAME,)from posthog.temporal.ai_observability.metrics import ExecutionTimeRecorder, get_metric_meter+from posthog.temporal.common.metrics import record_schedule_to_start_latency# ---------------------------------------------------------------------------# Histogram bucket config (imported by common/worker.py for PrometheusConfig)@@ -153,16 +153,13 @@ class _ClusteringActivityInterceptor(ActivityInboundInterceptor):return await super().execute_activity(input)# Schedule-to-start latency (queue depth indicator)- scheduled_time = activity_info.scheduled_time- started_time = activity_info.started_time- if scheduled_time and started_time:- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)- meter = get_metric_meter({"activity_type": activity_type})- meter.create_histogram_timedelta(- name="llma_clustering_activity_schedule_to_start_latency",- description="Time between activity scheduling and start",- unit="ms",- ).record(dt.timedelta(milliseconds=schedule_to_start_ms))+ record_schedule_to_start_latency(+ "llma_clustering_activity_schedule_to_start_latency",+ scheduled_time=activity_info.scheduled_time,+ started_time=activity_info.started_time,+ description="Time between activity scheduling and start",+ attributes={"activity_type": activity_type},+ )with ExecutionTimeRecorder("llma_clustering_activity_execution_latency",diff --git a/posthog/temporal/ai_observability/trace_summarization/metrics.py b/posthog/temporal/ai_observability/trace_summarization/metrics.pyindex 799ea4f42002..6fda13c506ad 100644--- a/posthog/temporal/ai_observability/trace_summarization/metrics.py+++ b/posthog/temporal/ai_observability/trace_summarization/metrics.py@@ -6,7 +6,6 @@ and scraped by the Prometheus endpoint on the worker pod."""import typing-import datetime as dtfrom django.conf import settings@@ -21,6 +20,7 @@ from temporalio.worker import ()from posthog.temporal.ai_observability.metrics import ExecutionTimeRecorder, get_metric_meter+from posthog.temporal.common.metrics import record_schedule_to_start_latency# ---------------------------------------------------------------------------# Activity / workflow type sets for the interceptor@@ -129,16 +129,13 @@ class _SummarizationActivityInterceptor(ActivityInboundInterceptor):return await super().execute_activity(input)# Schedule-to-start latency (queue depth indicator)- scheduled_time = activity_info.scheduled_time- started_time = activity_info.started_time- if scheduled_time and started_time:- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)- meter = get_metric_meter({"activity_type": activity_type})- meter.create_histogram_timedelta(- name="llma_summarization_activity_schedule_to_start_latency",- description="Time between activity scheduling and start",- unit="ms",- ).record(dt.timedelta(milliseconds=schedule_to_start_ms))+ record_schedule_to_start_latency(+ "llma_summarization_activity_schedule_to_start_latency",+ scheduled_time=activity_info.scheduled_time,+ started_time=activity_info.started_time,+ description="Time between activity scheduling and start",+ attributes={"activity_type": activity_type},+ )with ExecutionTimeRecorder("llma_summarization_activity_execution_latency",diff --git a/posthog/temporal/common/metrics.py b/posthog/temporal/common/metrics.pyindex 21c349201f29..33534a28911f 100644--- a/posthog/temporal/common/metrics.py+++ b/posthog/temporal/common/metrics.py@@ -25,6 +25,30 @@ def get_metric_meter(additional_attributes: Attributes | None = None) -> MetricMreturn meter+def record_schedule_to_start_latency(+ histogram_name: str,+ /,+ scheduled_time: dt.datetime | None,+ started_time: dt.datetime | None,+ description: str,+ attributes: Attributes | None = None,+) -> None:+ """Record the time an activity waited in the task queue to a histogram metric.++ The worker clock can disagree with the server clock, so the raw difference can be negative.+ Temporal raises on a negative metric value, so clamp it to zero. Like `ExecutionTimeRecorder`,+ this is best-effort: a metric failure must never fail the activity.+ """+ if scheduled_time is None or started_time is None:+ return+ latency = max(started_time - scheduled_time, dt.timedelta(0))+ try:+ meter = get_metric_meter(attributes)+ meter.create_histogram_timedelta(name=histogram_name, description=description, unit="ms").record(latency)+ except Exception:+ LOGGER.exception("Failed to record schedule-to-start latency to histogram '%s'", histogram_name)++class ExecutionTimeRecorder:"""Context manager to record execution time to a histogram metric."""diff --git a/posthog/temporal/common/test_metrics.py b/posthog/temporal/common/test_metrics.pynew file mode 100644index 000000000000..61637edc03d9--- /dev/null+++ b/posthog/temporal/common/test_metrics.py@@ -0,0 +1,45 @@+import datetime as dt++import pytest+from unittest.mock import MagicMock, patch++from posthog.temporal.common.metrics import record_schedule_to_start_latency+++class TestRecordScheduleToStartLatency:+ @pytest.mark.parametrize(+ "started_offset,expected",+ [+ (dt.timedelta(milliseconds=250), dt.timedelta(milliseconds=250)),+ (dt.timedelta(seconds=-3), dt.timedelta(0)),+ ],+ ids=["positive", "negative_clock_skew"],+ )+ def test_records_latency_clamped_to_zero(self, started_offset, expected):+ scheduled = dt.datetime(2026, 1, 1, tzinfo=dt.UTC)+ mock_meter = MagicMock()++ with patch("posthog.temporal.common.metrics.get_metric_meter", return_value=mock_meter):+ record_schedule_to_start_latency(+ "test_histogram",+ scheduled_time=scheduled,+ started_time=scheduled + started_offset,+ description="test",+ )++ mock_meter.create_histogram_timedelta.return_value.record.assert_called_once_with(expected)++ def test_metric_failure_does_not_raise(self):+ mock_meter = MagicMock()+ mock_meter.create_histogram_timedelta.return_value.record.side_effect = ValueError(+ "Metric value cannot be negative"+ )+ scheduled = dt.datetime(2026, 1, 1, tzinfo=dt.UTC)++ with patch("posthog.temporal.common.metrics.get_metric_meter", return_value=mock_meter):+ record_schedule_to_start_latency(+ "test_histogram",+ scheduled_time=scheduled,+ started_time=scheduled,+ description="test",+ )diff --git a/posthog/temporal/mcp_analytics/intent_clustering/metrics.py b/posthog/temporal/mcp_analytics/intent_clustering/metrics.pyindex 766743152260..6ad123af6633 100644--- a/posthog/temporal/mcp_analytics/intent_clustering/metrics.py+++ b/posthog/temporal/mcp_analytics/intent_clustering/metrics.py@@ -27,6 +27,7 @@ from temporalio.worker import ()from posthog.temporal.ai_observability.metrics import ExecutionTimeRecorder, get_metric_meter+from posthog.temporal.common.metrics import record_schedule_to_start_latencyfrom posthog.temporal.mcp_analytics.intent_clustering.constants import COORDINATOR_WORKFLOW_NAME, WORKFLOW_NAME# Histograms registered into PrometheusConfig via common/worker.py.@@ -117,15 +118,13 @@ class _MCPAClusteringActivityInterceptor(ActivityInboundInterceptor):return await super().execute_activity(input)# Schedule-to-start latency (queue depth indicator).- scheduled = activity_info.scheduled_time- started = activity_info.started_time- if scheduled and started:- meter = get_metric_meter({"activity_type": activity_type})- meter.create_histogram_timedelta(- name="mcpa_clustering_activity_schedule_to_start_latency",- description="Time between activity scheduling and start",- unit="ms",- ).record(started - scheduled)+ record_schedule_to_start_latency(+ "mcpa_clustering_activity_schedule_to_start_latency",+ scheduled_time=activity_info.scheduled_time,+ started_time=activity_info.started_time,+ description="Time between activity scheduling and start",+ attributes={"activity_type": activity_type},+ )with ExecutionTimeRecorder("mcpa_clustering_activity_execution_latency",diff --git a/posthog/temporal/session_replay/delete_recordings/metrics.py b/posthog/temporal/session_replay/delete_recordings/metrics.pyindex 4fc4db90eeff..60410452f4ef 100644--- a/posthog/temporal/session_replay/delete_recordings/metrics.py+++ b/posthog/temporal/session_replay/delete_recordings/metrics.py@@ -1,5 +1,4 @@import typing-import datetime as dtfrom django.conf import settings@@ -13,7 +12,7 @@ from temporalio.worker import (WorkflowInterceptorClassInput,)-from posthog.temporal.common.metrics import ExecutionTimeRecorder, get_metric_meter+from posthog.temporal.common.metrics import ExecutionTimeRecorder, get_metric_meter, record_schedule_to_start_latency# ---------------------------------------------------------------------------# Histogram bucket config (imported by common/worker.py for PrometheusConfig)@@ -109,16 +108,12 @@ class _DeleteRecordingsActivityInterceptor(ActivityInboundInterceptor):if activity_type not in DELETE_RECORDINGS_ACTIVITY_TYPES:return await super().execute_activity(input)- scheduled_time = activity_info.scheduled_time- started_time = activity_info.started_time- if scheduled_time and started_time:- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)- meter = get_metric_meter()- meter.create_histogram_timedelta(- name="delete_recordings_activity_schedule_to_start_latency",- description="Time between activity scheduling and start",- unit="ms",- ).record(dt.timedelta(milliseconds=schedule_to_start_ms))+ record_schedule_to_start_latency(+ "delete_recordings_activity_schedule_to_start_latency",+ scheduled_time=activity_info.scheduled_time,+ started_time=activity_info.started_time,+ description="Time between activity scheduling and start",+ )with ExecutionTimeRecorder("delete_recordings_activity_execution_latency",diff --git a/products/experiments/backend/temporal/recalculation_metrics.py b/products/experiments/backend/temporal/recalculation_metrics.pyindex f3cb318f836e..dbf05b093a33 100644--- a/products/experiments/backend/temporal/recalculation_metrics.py+++ b/products/experiments/backend/temporal/recalculation_metrics.py@@ -23,6 +23,7 @@ from temporalio.worker import ()from posthog.temporal.ai_observability.metrics import ExecutionTimeRecorder+from posthog.temporal.common.metrics import record_schedule_to_start_latencyfrom products.experiments.backend.temporal.models import METRICS_RECALCULATION_WORKFLOW_NAME@@ -127,16 +128,19 @@ class _ActivityInboundInterceptor(ActivityInboundInterceptor):)# Queue-pressure signal (see bucket comment above). Per-attempt scheduling time, so retry backoff# (including the intentional quota-wait delays) doesn't read as queue pressure.- if info.current_attempt_scheduled_time and info.started_time:- # A retry attempt waits on the queue after its backoff, so splitting it out separates retry waves- # from fresh fan-out that waits for a free slot.- meter.with_additional_attributes(- {"attempt_kind": "first" if info.attempt == 1 else "retry"}- ).create_histogram_timedelta(- name="experiment_metrics_recalculation_activity_schedule_to_start_latency",- description="Time between the current attempt's scheduling and start (task queue wait).",- unit="ms",- ).record(info.started_time - info.current_attempt_scheduled_time)+ # A retry attempt waits on the queue after its backoff, so splitting it out separates retry waves+ # from fresh fan-out that waits for a free slot.+ record_schedule_to_start_latency(+ "experiment_metrics_recalculation_activity_schedule_to_start_latency",+ scheduled_time=info.current_attempt_scheduled_time,+ started_time=info.started_time,+ description="Time between the current attempt's scheduling and start (task queue wait).",+ attributes={+ "activity_type": activity_type,+ "workflow_type": METRICS_RECALCULATION_WORKFLOW_NAME,+ "attempt_kind": "first" if info.attempt == 1 else "retry",+ },+ )# Fires on every attempt (first run + each retry). Comparing this to `_activity_successes` reveals# retry rate; without it a transient ClickHouse blip is indistinguishable from a healthy first-try# success. Pattern mirrors batch_exports' `batch_exports_activity_attempts`.diff --git a/products/logs/backend/temporal/metrics.py b/products/logs/backend/temporal/metrics.pyindex dd7d881e2ec3..4c20ac8b76d4 100644--- a/products/logs/backend/temporal/metrics.py+++ b/products/logs/backend/temporal/metrics.py@@ -13,6 +13,7 @@ from temporalio.common import MetricMeterfrom temporalio.worker import ActivityInboundInterceptor, ExecuteActivityInput, Interceptorfrom posthog.temporal.common.logger import get_write_only_logger+from posthog.temporal.common.metrics import record_schedule_to_start_latencyfrom products.logs.backend.alert_error_classifier import AlertErrorCodefrom products.logs.backend.alert_state_machine import AlertState, NotificationAction@@ -274,15 +275,6 @@ def record_scheduler_lag(lag_ms: int) -> None:_record_histogram("logs_alerting_scheduler_lag_ms", "Delay between alert due time and actual check time", lag_ms)-def record_schedule_to_start_latency(activity_type: str, latency_ms: int) -> None:- _record_histogram(- "logs_alerting_schedule_to_start_ms",- "Time between activity scheduling and start",- latency_ms,- {"activity_type": activity_type},- )--# TODO: Extract ExecutionTimeRecorder to posthog/temporal/common/ — copied from# posthog/temporal/ai_observability/metrics.py to avoid cross-product import.class ExecutionTimeRecorder:@@ -345,11 +337,13 @@ class _LogsAlertingActivityInterceptor(ActivityInboundInterceptor):if activity_type not in ALERTING_ACTIVITY_TYPES:return await super().execute_activity(input)- scheduled_time = activity_info.scheduled_time- started_time = activity_info.started_time- if scheduled_time and started_time:- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)- record_schedule_to_start_latency(activity_type, schedule_to_start_ms)+ record_schedule_to_start_latency(+ "logs_alerting_schedule_to_start_ms",+ scheduled_time=activity_info.scheduled_time,+ started_time=activity_info.started_time,+ description="Time between activity scheduling and start",+ attributes={"activity_type": activity_type},+ )with ExecutionTimeRecorder("logs_alerting_cycle_duration_ms",