Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): clamp negative schedule-to-start latency in temporal interceptors (#122478)

posthog[bot]f201bfad0927
diff --git a/posthog/temporal/ai_observability/eval_reports/metrics.py b/posthog/temporal/ai_observability/eval_reports/metrics.py
index ff1439698ad2..5488bb6b860e 100644
--- a/posthog/temporal/ai_observability/eval_reports/metrics.py
+++ b/posthog/temporal/ai_observability/eval_reports/metrics.py
@@ -6,7 +6,6 @@ and scraped by the Prometheus endpoint on the worker pod.
"""
import typing
-import datetime as dt
from django.conf import settings
@@ -21,6 +20,7 @@ from temporalio.worker import (
)
from posthog.temporal.ai_observability.metrics import ExecutionTimeRecorder, get_metric_meter
+from posthog.temporal.common.metrics import record_schedule_to_start_latency
# ---------------------------------------------------------------------------
# Histogram bucket config (imported by common/worker.py for PrometheusConfig)
@@ -175,16 +175,13 @@ class _EvalReportsActivityInterceptor(ActivityInboundInterceptor):
return await super().execute_activity(input)
# Schedule-to-start latency (queue depth indicator)
- scheduled_time = activity_info.scheduled_time
- started_time = activity_info.started_time
- if scheduled_time and started_time:
- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)
- meter = get_metric_meter({"activity_type": activity_type})
- meter.create_histogram_timedelta(
- name="llma_eval_reports_activity_schedule_to_start_latency",
- description="Time between activity scheduling and start",
- unit="ms",
- ).record(dt.timedelta(milliseconds=schedule_to_start_ms))
+ record_schedule_to_start_latency(
+ "llma_eval_reports_activity_schedule_to_start_latency",
+ scheduled_time=activity_info.scheduled_time,
+ started_time=activity_info.started_time,
+ description="Time between activity scheduling and start",
+ attributes={"activity_type": activity_type},
+ )
with ExecutionTimeRecorder(
"llma_eval_reports_activity_execution_latency",
diff --git a/posthog/temporal/ai_observability/metrics.py b/posthog/temporal/ai_observability/metrics.py
index 7ee64cc752d7..98c1b97fa19a 100644
--- a/posthog/temporal/ai_observability/metrics.py
+++ b/posthog/temporal/ai_observability/metrics.py
@@ -1,5 +1,4 @@
import typing
-import datetime as dt
from django.conf import settings
@@ -17,7 +16,12 @@ from posthog.temporal.common.logger import get_write_only_logger
# Generic temporal metric helpers live in common so other subsystems (e.g. session-replay
# delete-recordings) can import them without pulling this package's eager worker registration.
-from posthog.temporal.common.metrics import Attributes, ExecutionTimeRecorder, get_metric_meter
+from posthog.temporal.common.metrics import (
+ Attributes,
+ ExecutionTimeRecorder,
+ get_metric_meter,
+ record_schedule_to_start_latency,
+)
__all__ = ["Attributes", "ExecutionTimeRecorder", "get_metric_meter"]
@@ -175,17 +179,6 @@ def increment_backfill_remainder_outcome(outcome: str) -> None:
counter.add(1)
-def record_schedule_to_start_latency(activity_type: str, latency_ms: int) -> None:
- """Record queue depth indicator for alerting."""
- meter = get_metric_meter({"activity_type": activity_type})
- hist = meter.create_histogram_timedelta(
- name="llma_eval_activity_schedule_to_start_latency",
- description="Time between activity scheduling and start (queue depth indicator)",
- unit="ms",
- )
- hist.record(dt.timedelta(milliseconds=latency_ms))
-
-
class EvalsMetricsInterceptor(Interceptor):
"""Interceptor to emit Prometheus metrics for evals workflows."""
@@ -209,11 +202,13 @@ class _EvalsMetricsActivityInboundInterceptor(ActivityInboundInterceptor):
return await super().execute_activity(input)
# Record schedule-to-start latency (queue depth indicator)
- scheduled_time = activity_info.scheduled_time
- started_time = activity_info.started_time
- if scheduled_time and started_time:
- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)
- record_schedule_to_start_latency(activity_type, schedule_to_start_ms)
+ record_schedule_to_start_latency(
+ "llma_eval_activity_schedule_to_start_latency",
+ scheduled_time=activity_info.scheduled_time,
+ started_time=activity_info.started_time,
+ description="Time between activity scheduling and start (queue depth indicator)",
+ attributes={"activity_type": activity_type},
+ )
histogram_attributes: Attributes = {"activity_type": activity_type}
diff --git a/posthog/temporal/ai_observability/trace_clustering/metrics.py b/posthog/temporal/ai_observability/trace_clustering/metrics.py
index b190d4cfcbad..52a9b2c508d0 100644
--- a/posthog/temporal/ai_observability/trace_clustering/metrics.py
+++ b/posthog/temporal/ai_observability/trace_clustering/metrics.py
@@ -6,7 +6,6 @@ and scraped by the Prometheus endpoint on the worker pod.
"""
import typing
-import datetime as dt
from django.conf import settings
@@ -24,6 +23,7 @@ from posthog.temporal.ai_observability.evaluation_clustering.constants import (
CLUSTERING_WORKFLOW_NAME as EVAL_CLUSTERING_WORKFLOW_NAME,
)
from posthog.temporal.ai_observability.metrics import ExecutionTimeRecorder, get_metric_meter
+from posthog.temporal.common.metrics import record_schedule_to_start_latency
# ---------------------------------------------------------------------------
# Histogram bucket config (imported by common/worker.py for PrometheusConfig)
@@ -153,16 +153,13 @@ class _ClusteringActivityInterceptor(ActivityInboundInterceptor):
return await super().execute_activity(input)
# Schedule-to-start latency (queue depth indicator)
- scheduled_time = activity_info.scheduled_time
- started_time = activity_info.started_time
- if scheduled_time and started_time:
- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)
- meter = get_metric_meter({"activity_type": activity_type})
- meter.create_histogram_timedelta(
- name="llma_clustering_activity_schedule_to_start_latency",
- description="Time between activity scheduling and start",
- unit="ms",
- ).record(dt.timedelta(milliseconds=schedule_to_start_ms))
+ record_schedule_to_start_latency(
+ "llma_clustering_activity_schedule_to_start_latency",
+ scheduled_time=activity_info.scheduled_time,
+ started_time=activity_info.started_time,
+ description="Time between activity scheduling and start",
+ attributes={"activity_type": activity_type},
+ )
with ExecutionTimeRecorder(
"llma_clustering_activity_execution_latency",
diff --git a/posthog/temporal/ai_observability/trace_summarization/metrics.py b/posthog/temporal/ai_observability/trace_summarization/metrics.py
index 799ea4f42002..6fda13c506ad 100644
--- a/posthog/temporal/ai_observability/trace_summarization/metrics.py
+++ b/posthog/temporal/ai_observability/trace_summarization/metrics.py
@@ -6,7 +6,6 @@ and scraped by the Prometheus endpoint on the worker pod.
"""
import typing
-import datetime as dt
from django.conf import settings
@@ -21,6 +20,7 @@ from temporalio.worker import (
)
from posthog.temporal.ai_observability.metrics import ExecutionTimeRecorder, get_metric_meter
+from posthog.temporal.common.metrics import record_schedule_to_start_latency
# ---------------------------------------------------------------------------
# Activity / workflow type sets for the interceptor
@@ -129,16 +129,13 @@ class _SummarizationActivityInterceptor(ActivityInboundInterceptor):
return await super().execute_activity(input)
# Schedule-to-start latency (queue depth indicator)
- scheduled_time = activity_info.scheduled_time
- started_time = activity_info.started_time
- if scheduled_time and started_time:
- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)
- meter = get_metric_meter({"activity_type": activity_type})
- meter.create_histogram_timedelta(
- name="llma_summarization_activity_schedule_to_start_latency",
- description="Time between activity scheduling and start",
- unit="ms",
- ).record(dt.timedelta(milliseconds=schedule_to_start_ms))
+ record_schedule_to_start_latency(
+ "llma_summarization_activity_schedule_to_start_latency",
+ scheduled_time=activity_info.scheduled_time,
+ started_time=activity_info.started_time,
+ description="Time between activity scheduling and start",
+ attributes={"activity_type": activity_type},
+ )
with ExecutionTimeRecorder(
"llma_summarization_activity_execution_latency",
diff --git a/posthog/temporal/common/metrics.py b/posthog/temporal/common/metrics.py
index 21c349201f29..33534a28911f 100644
--- a/posthog/temporal/common/metrics.py
+++ b/posthog/temporal/common/metrics.py
@@ -25,6 +25,30 @@ def get_metric_meter(additional_attributes: Attributes | None = None) -> MetricM
return meter
+def record_schedule_to_start_latency(
+ histogram_name: str,
+ /,
+ scheduled_time: dt.datetime | None,
+ started_time: dt.datetime | None,
+ description: str,
+ attributes: Attributes | None = None,
+) -> None:
+ """Record the time an activity waited in the task queue to a histogram metric.
+
+ The worker clock can disagree with the server clock, so the raw difference can be negative.
+ Temporal raises on a negative metric value, so clamp it to zero. Like `ExecutionTimeRecorder`,
+ this is best-effort: a metric failure must never fail the activity.
+ """
+ if scheduled_time is None or started_time is None:
+ return
+ latency = max(started_time - scheduled_time, dt.timedelta(0))
+ try:
+ meter = get_metric_meter(attributes)
+ meter.create_histogram_timedelta(name=histogram_name, description=description, unit="ms").record(latency)
+ except Exception:
+ LOGGER.exception("Failed to record schedule-to-start latency to histogram '%s'", histogram_name)
+
+
class ExecutionTimeRecorder:
"""Context manager to record execution time to a histogram metric."""
diff --git a/posthog/temporal/common/test_metrics.py b/posthog/temporal/common/test_metrics.py
new file mode 100644
index 000000000000..61637edc03d9
--- /dev/null
+++ b/posthog/temporal/common/test_metrics.py
@@ -0,0 +1,45 @@
+import datetime as dt
+
+import pytest
+from unittest.mock import MagicMock, patch
+
+from posthog.temporal.common.metrics import record_schedule_to_start_latency
+
+
+class TestRecordScheduleToStartLatency:
+ @pytest.mark.parametrize(
+ "started_offset,expected",
+ [
+ (dt.timedelta(milliseconds=250), dt.timedelta(milliseconds=250)),
+ (dt.timedelta(seconds=-3), dt.timedelta(0)),
+ ],
+ ids=["positive", "negative_clock_skew"],
+ )
+ def test_records_latency_clamped_to_zero(self, started_offset, expected):
+ scheduled = dt.datetime(2026, 1, 1, tzinfo=dt.UTC)
+ mock_meter = MagicMock()
+
+ with patch("posthog.temporal.common.metrics.get_metric_meter", return_value=mock_meter):
+ record_schedule_to_start_latency(
+ "test_histogram",
+ scheduled_time=scheduled,
+ started_time=scheduled + started_offset,
+ description="test",
+ )
+
+ mock_meter.create_histogram_timedelta.return_value.record.assert_called_once_with(expected)
+
+ def test_metric_failure_does_not_raise(self):
+ mock_meter = MagicMock()
+ mock_meter.create_histogram_timedelta.return_value.record.side_effect = ValueError(
+ "Metric value cannot be negative"
+ )
+ scheduled = dt.datetime(2026, 1, 1, tzinfo=dt.UTC)
+
+ with patch("posthog.temporal.common.metrics.get_metric_meter", return_value=mock_meter):
+ record_schedule_to_start_latency(
+ "test_histogram",
+ scheduled_time=scheduled,
+ started_time=scheduled,
+ description="test",
+ )
diff --git a/posthog/temporal/mcp_analytics/intent_clustering/metrics.py b/posthog/temporal/mcp_analytics/intent_clustering/metrics.py
index 766743152260..6ad123af6633 100644
--- a/posthog/temporal/mcp_analytics/intent_clustering/metrics.py
+++ b/posthog/temporal/mcp_analytics/intent_clustering/metrics.py
@@ -27,6 +27,7 @@ from temporalio.worker import (
)
from posthog.temporal.ai_observability.metrics import ExecutionTimeRecorder, get_metric_meter
+from posthog.temporal.common.metrics import record_schedule_to_start_latency
from posthog.temporal.mcp_analytics.intent_clustering.constants import COORDINATOR_WORKFLOW_NAME, WORKFLOW_NAME
# Histograms registered into PrometheusConfig via common/worker.py.
@@ -117,15 +118,13 @@ class _MCPAClusteringActivityInterceptor(ActivityInboundInterceptor):
return await super().execute_activity(input)
# Schedule-to-start latency (queue depth indicator).
- scheduled = activity_info.scheduled_time
- started = activity_info.started_time
- if scheduled and started:
- meter = get_metric_meter({"activity_type": activity_type})
- meter.create_histogram_timedelta(
- name="mcpa_clustering_activity_schedule_to_start_latency",
- description="Time between activity scheduling and start",
- unit="ms",
- ).record(started - scheduled)
+ record_schedule_to_start_latency(
+ "mcpa_clustering_activity_schedule_to_start_latency",
+ scheduled_time=activity_info.scheduled_time,
+ started_time=activity_info.started_time,
+ description="Time between activity scheduling and start",
+ attributes={"activity_type": activity_type},
+ )
with ExecutionTimeRecorder(
"mcpa_clustering_activity_execution_latency",
diff --git a/posthog/temporal/session_replay/delete_recordings/metrics.py b/posthog/temporal/session_replay/delete_recordings/metrics.py
index 4fc4db90eeff..60410452f4ef 100644
--- a/posthog/temporal/session_replay/delete_recordings/metrics.py
+++ b/posthog/temporal/session_replay/delete_recordings/metrics.py
@@ -1,5 +1,4 @@
import typing
-import datetime as dt
from django.conf import settings
@@ -13,7 +12,7 @@ from temporalio.worker import (
WorkflowInterceptorClassInput,
)
-from posthog.temporal.common.metrics import ExecutionTimeRecorder, get_metric_meter
+from posthog.temporal.common.metrics import ExecutionTimeRecorder, get_metric_meter, record_schedule_to_start_latency
# ---------------------------------------------------------------------------
# Histogram bucket config (imported by common/worker.py for PrometheusConfig)
@@ -109,16 +108,12 @@ class _DeleteRecordingsActivityInterceptor(ActivityInboundInterceptor):
if activity_type not in DELETE_RECORDINGS_ACTIVITY_TYPES:
return await super().execute_activity(input)
- scheduled_time = activity_info.scheduled_time
- started_time = activity_info.started_time
- if scheduled_time and started_time:
- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)
- meter = get_metric_meter()
- meter.create_histogram_timedelta(
- name="delete_recordings_activity_schedule_to_start_latency",
- description="Time between activity scheduling and start",
- unit="ms",
- ).record(dt.timedelta(milliseconds=schedule_to_start_ms))
+ record_schedule_to_start_latency(
+ "delete_recordings_activity_schedule_to_start_latency",
+ scheduled_time=activity_info.scheduled_time,
+ started_time=activity_info.started_time,
+ description="Time between activity scheduling and start",
+ )
with ExecutionTimeRecorder(
"delete_recordings_activity_execution_latency",
diff --git a/products/experiments/backend/temporal/recalculation_metrics.py b/products/experiments/backend/temporal/recalculation_metrics.py
index f3cb318f836e..dbf05b093a33 100644
--- a/products/experiments/backend/temporal/recalculation_metrics.py
+++ b/products/experiments/backend/temporal/recalculation_metrics.py
@@ -23,6 +23,7 @@ from temporalio.worker import (
)
from posthog.temporal.ai_observability.metrics import ExecutionTimeRecorder
+from posthog.temporal.common.metrics import record_schedule_to_start_latency
from products.experiments.backend.temporal.models import METRICS_RECALCULATION_WORKFLOW_NAME
@@ -127,16 +128,19 @@ class _ActivityInboundInterceptor(ActivityInboundInterceptor):
)
# Queue-pressure signal (see bucket comment above). Per-attempt scheduling time, so retry backoff
# (including the intentional quota-wait delays) doesn't read as queue pressure.
- if info.current_attempt_scheduled_time and info.started_time:
- # A retry attempt waits on the queue after its backoff, so splitting it out separates retry waves
- # from fresh fan-out that waits for a free slot.
- meter.with_additional_attributes(
- {"attempt_kind": "first" if info.attempt == 1 else "retry"}
- ).create_histogram_timedelta(
- name="experiment_metrics_recalculation_activity_schedule_to_start_latency",
- description="Time between the current attempt's scheduling and start (task queue wait).",
- unit="ms",
- ).record(info.started_time - info.current_attempt_scheduled_time)
+ # A retry attempt waits on the queue after its backoff, so splitting it out separates retry waves
+ # from fresh fan-out that waits for a free slot.
+ record_schedule_to_start_latency(
+ "experiment_metrics_recalculation_activity_schedule_to_start_latency",
+ scheduled_time=info.current_attempt_scheduled_time,
+ started_time=info.started_time,
+ description="Time between the current attempt's scheduling and start (task queue wait).",
+ attributes={
+ "activity_type": activity_type,
+ "workflow_type": METRICS_RECALCULATION_WORKFLOW_NAME,
+ "attempt_kind": "first" if info.attempt == 1 else "retry",
+ },
+ )
# Fires on every attempt (first run + each retry). Comparing this to `_activity_successes` reveals
# retry rate; without it a transient ClickHouse blip is indistinguishable from a healthy first-try
# success. Pattern mirrors batch_exports' `batch_exports_activity_attempts`.
diff --git a/products/logs/backend/temporal/metrics.py b/products/logs/backend/temporal/metrics.py
index dd7d881e2ec3..4c20ac8b76d4 100644
--- a/products/logs/backend/temporal/metrics.py
+++ b/products/logs/backend/temporal/metrics.py
@@ -13,6 +13,7 @@ from temporalio.common import MetricMeter
from temporalio.worker import ActivityInboundInterceptor, ExecuteActivityInput, Interceptor
from posthog.temporal.common.logger import get_write_only_logger
+from posthog.temporal.common.metrics import record_schedule_to_start_latency
from products.logs.backend.alert_error_classifier import AlertErrorCode
from products.logs.backend.alert_state_machine import AlertState, NotificationAction
@@ -274,15 +275,6 @@ def record_scheduler_lag(lag_ms: int) -> None:
_record_histogram("logs_alerting_scheduler_lag_ms", "Delay between alert due time and actual check time", lag_ms)
-def record_schedule_to_start_latency(activity_type: str, latency_ms: int) -> None:
- _record_histogram(
- "logs_alerting_schedule_to_start_ms",
- "Time between activity scheduling and start",
- latency_ms,
- {"activity_type": activity_type},
- )
-
-
# TODO: Extract ExecutionTimeRecorder to posthog/temporal/common/ — copied from
# posthog/temporal/ai_observability/metrics.py to avoid cross-product import.
class ExecutionTimeRecorder:
@@ -345,11 +337,13 @@ class _LogsAlertingActivityInterceptor(ActivityInboundInterceptor):
if activity_type not in ALERTING_ACTIVITY_TYPES:
return await super().execute_activity(input)
- scheduled_time = activity_info.scheduled_time
- started_time = activity_info.started_time
- if scheduled_time and started_time:
- schedule_to_start_ms = int((started_time - scheduled_time).total_seconds() * 1000)
- record_schedule_to_start_latency(activity_type, schedule_to_start_ms)
+ record_schedule_to_start_latency(
+ "logs_alerting_schedule_to_start_ms",
+ scheduled_time=activity_info.scheduled_time,
+ started_time=activity_info.started_time,
+ description="Time between activity scheduling and start",
+ attributes={"activity_type": activity_type},
+ )
with ExecutionTimeRecorder(
"logs_alerting_cycle_duration_ms",