:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(logs): measure platform logs scheduler lag on the wall clock (#124475)
Jimmy Zhu0bdf416073f2
diff --git a/products/logs/backend/alert_source_cycle.py b/products/logs/backend/alert_source_cycle.pyindex 4b55b4ff9f21..ced24bd7a9e6 100644--- a/products/logs/backend/alert_source_cycle.py+++ b/products/logs/backend/alert_source_cycle.py@@ -18,7 +18,7 @@ import jsonimport timefrom collections.abc import Mapping, Sequencefrom dataclasses import replace-from datetime import datetime, timedelta+from datetime import UTC, datetime, timedeltafrom itertools import batchedfrom typing import Any, Final, castfrom uuid import UUID, uuid4@@ -310,9 +310,7 @@ def _record_check_metrics(muted_notification: NotificationAction = NotificationAction.NONE,skip: SkipReason | None = None,muted_by_quiet_hours: bool = False,- now: datetime,grouping_key: str = "",- record_lag: bool = True,) -> None:safe_record(increment_checks, SourceKind.LOGS.value, notification.value)if skip is not None:@@ -324,8 +322,17 @@ def _record_check_metrics(previous_state = check.instance(grouping_key).stateif previous_state != new_state:safe_record(increment_state_transition, SourceKind.LOGS.value, previous_state, new_state)- if record_lag and check.next_check_at is not None:- lag_ms = int((now - check.next_check_at).total_seconds() * 1000)+++def _record_scheduler_lag(checks: Sequence[PlatformAlertCheckInput]) -> None:+ # Wall clock rather than the tick: due times and the tick both fall on the minute, so lag+ # against the tick is 0 until a check misses a whole tick, and it hides the wait in dispatch,+ # the worker queue and the batch pool.+ taken_at = datetime.now(UTC)+ for check in checks:+ if check.next_check_at is None:+ continue+ lag_ms = int((taken_at - check.next_check_at).total_seconds() * 1000)if lag_ms > 0:safe_record(record_scheduler_lag, SourceKind.LOGS.value, lag_ms)@@ -358,7 +365,6 @@ def _verdict(muted_notification=outcome.muted_notification,skip=skip,muted_by_quiet_hours=check_input.muted,- now=now,)return outcome@@ -570,9 +576,7 @@ def _held(error_message=error_message,failed=True,)- _record_check_metrics(- check, new_state=outcome.new_state.value, notification=NotificationAction.NONE, skip=skip, now=now- )+ _record_check_metrics(check, new_state=outcome.new_state.value, notification=NotificationAction.NONE, skip=skip)return recorded, _request(check, recorded, _ungrouped_move(check, outcome), sends_messages=False)@@ -736,9 +740,7 @@ def _stopped(check: PlatformAlertCheckInput, *, now: datetime) -> Decision:consecutive_failures=check.consecutive_failures,disable=True,)- _record_check_metrics(- check, new_state=state, notification=NotificationAction.NONE, skip=SkipReason.SOURCE_RULE, now=now- )+ _record_check_metrics(check, new_state=state, notification=NotificationAction.NONE, skip=SkipReason.SOURCE_RULE)return recorded, None@@ -842,16 +844,14 @@ def _evaluate_grouped(muted=muted,absent=absent,)- for index, verdict in enumerate(decision.verdicts):+ for verdict in decision.verdicts:_record_check_metrics(check,new_state=verdict.group.new_state,notification=verdict.outcome.notification,muted_notification=verdict.outcome.muted_notification,muted_by_quiet_hours=muted,- now=now,grouping_key=verdict.group.grouping_key,- record_lag=index == 0,)recorded = decision.as_outcome(configuration_id=check.id,@@ -1064,6 +1064,7 @@ def _evaluate_admitted(if team is None:return SourceBatchEvaluation(outcomes=(), deliveries=())+ _record_scheduler_lag(checks)current, stopped = _as_configured_now(checks, team_id=team_id, now=cutoff)triage = _triage(current, now=cutoff, tz_name=team.timezone)if not triage.evaluable:diff --git a/products/logs/backend/test/test_alert_evaluate.py b/products/logs/backend/test/test_alert_evaluate.pyindex 3a7255b83187..cb480705df6e 100644--- a/products/logs/backend/test/test_alert_evaluate.py+++ b/products/logs/backend/test/test_alert_evaluate.py@@ -211,6 +211,17 @@ class TestLogsAlertEvaluation(APIBaseTest):assert configuration.next_check_at is not Noneassert configuration.next_check_at > self.cutoff+ def test_scheduler_lag_counts_the_wait_after_the_tick_that_took_the_check(self) -> None:+ configuration = self._configuration(next_check_at=self.cutoff)++ with (+ time_machine.travel(self.cutoff + timedelta(seconds=7), tick=False),+ patch(f"{_MODULE}.record_scheduler_lag") as record_lag,+ ):+ self._run(configuration)++ record_lag.assert_called_once_with(SourceKind.LOGS.value, 7000)+def test_checks_that_can_share_a_query_carry_one_schedule_group(self) -> None:same_a = self._configuration(name="A")same_b = self._configuration(name="B")