Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(logs): measure platform logs scheduler lag on the wall clock (#124475)

Jimmy Zhu0bdf416073f2
diff --git a/products/logs/backend/alert_source_cycle.py b/products/logs/backend/alert_source_cycle.py
index 4b55b4ff9f21..ced24bd7a9e6 100644
--- a/products/logs/backend/alert_source_cycle.py
+++ b/products/logs/backend/alert_source_cycle.py
@@ -18,7 +18,7 @@ import json
import time
from collections.abc import Mapping, Sequence
from dataclasses import replace
-from datetime import datetime, timedelta
+from datetime import UTC, datetime, timedelta
from itertools import batched
from typing import Any, Final, cast
from uuid import UUID, uuid4
@@ -310,9 +310,7 @@ def _record_check_metrics(
muted_notification: NotificationAction = NotificationAction.NONE,
skip: SkipReason | None = None,
muted_by_quiet_hours: bool = False,
- now: datetime,
grouping_key: str = "",
- record_lag: bool = True,
) -> None:
safe_record(increment_checks, SourceKind.LOGS.value, notification.value)
if skip is not None:
@@ -324,8 +322,17 @@ def _record_check_metrics(
previous_state = check.instance(grouping_key).state
if previous_state != new_state:
safe_record(increment_state_transition, SourceKind.LOGS.value, previous_state, new_state)
- if record_lag and check.next_check_at is not None:
- lag_ms = int((now - check.next_check_at).total_seconds() * 1000)
+
+
+def _record_scheduler_lag(checks: Sequence[PlatformAlertCheckInput]) -> None:
+ # Wall clock rather than the tick: due times and the tick both fall on the minute, so lag
+ # against the tick is 0 until a check misses a whole tick, and it hides the wait in dispatch,
+ # the worker queue and the batch pool.
+ taken_at = datetime.now(UTC)
+ for check in checks:
+ if check.next_check_at is None:
+ continue
+ lag_ms = int((taken_at - check.next_check_at).total_seconds() * 1000)
if lag_ms > 0:
safe_record(record_scheduler_lag, SourceKind.LOGS.value, lag_ms)
@@ -358,7 +365,6 @@ def _verdict(
muted_notification=outcome.muted_notification,
skip=skip,
muted_by_quiet_hours=check_input.muted,
- now=now,
)
return outcome
@@ -570,9 +576,7 @@ def _held(
error_message=error_message,
failed=True,
)
- _record_check_metrics(
- check, new_state=outcome.new_state.value, notification=NotificationAction.NONE, skip=skip, now=now
- )
+ _record_check_metrics(check, new_state=outcome.new_state.value, notification=NotificationAction.NONE, skip=skip)
return recorded, _request(check, recorded, _ungrouped_move(check, outcome), sends_messages=False)
@@ -736,9 +740,7 @@ def _stopped(check: PlatformAlertCheckInput, *, now: datetime) -> Decision:
consecutive_failures=check.consecutive_failures,
disable=True,
)
- _record_check_metrics(
- check, new_state=state, notification=NotificationAction.NONE, skip=SkipReason.SOURCE_RULE, now=now
- )
+ _record_check_metrics(check, new_state=state, notification=NotificationAction.NONE, skip=SkipReason.SOURCE_RULE)
return recorded, None
@@ -842,16 +844,14 @@ def _evaluate_grouped(
muted=muted,
absent=absent,
)
- for index, verdict in enumerate(decision.verdicts):
+ for verdict in decision.verdicts:
_record_check_metrics(
check,
new_state=verdict.group.new_state,
notification=verdict.outcome.notification,
muted_notification=verdict.outcome.muted_notification,
muted_by_quiet_hours=muted,
- now=now,
grouping_key=verdict.group.grouping_key,
- record_lag=index == 0,
)
recorded = decision.as_outcome(
configuration_id=check.id,
@@ -1064,6 +1064,7 @@ def _evaluate_admitted(
if team is None:
return SourceBatchEvaluation(outcomes=(), deliveries=())
+ _record_scheduler_lag(checks)
current, stopped = _as_configured_now(checks, team_id=team_id, now=cutoff)
triage = _triage(current, now=cutoff, tz_name=team.timezone)
if not triage.evaluable:
diff --git a/products/logs/backend/test/test_alert_evaluate.py b/products/logs/backend/test/test_alert_evaluate.py
index 3a7255b83187..cb480705df6e 100644
--- a/products/logs/backend/test/test_alert_evaluate.py
+++ b/products/logs/backend/test/test_alert_evaluate.py
@@ -211,6 +211,17 @@ class TestLogsAlertEvaluation(APIBaseTest):
assert configuration.next_check_at is not None
assert configuration.next_check_at > self.cutoff
+ def test_scheduler_lag_counts_the_wait_after_the_tick_that_took_the_check(self) -> None:
+ configuration = self._configuration(next_check_at=self.cutoff)
+
+ with (
+ time_machine.travel(self.cutoff + timedelta(seconds=7), tick=False),
+ patch(f"{_MODULE}.record_scheduler_lag") as record_lag,
+ ):
+ self._run(configuration)
+
+ record_lag.assert_called_once_with(SourceKind.LOGS.value, 7000)
+
def test_checks_that_can_share_a_query_carry_one_schedule_group(self) -> None:
same_a = self._configuration(name="A")
same_b = self._configuration(name="B")