:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(logs): evaluate platform logs alerts that have no logs row (#128696)
Jimmy Zhu145fc8ddc07a
diff --git a/products/alerts_platform/backend/facade/api.py b/products/alerts_platform/backend/facade/api.pyindex 9132cbd60fd6..d1429e5fa1b0 100644--- a/products/alerts_platform/backend/facade/api.py+++ b/products/alerts_platform/backend/facade/api.py@@ -189,15 +189,17 @@ def list_configurations(limit: int,offset: int,legacy_configuration_id: UUID | None = None,+ configuration_ids: Collection[UUID] | None = None,) -> PlatformAlertConfigurationPage:"""One page of configurations a caller may read, with the total behind it. With a- `legacy_configuration_id`, only the copy of that source row."""+ `legacy_configuration_id`, only the copy of that source row. With `configuration_ids`, only those."""return platform_reads.list_configurations(team_id=team_id,source_kinds=source_kinds,limit=limit,offset=offset,legacy_configuration_id=legacy_configuration_id,+ configuration_ids=configuration_ids,)diff --git a/products/alerts_platform/backend/logic/platform_reads.py b/products/alerts_platform/backend/logic/platform_reads.pyindex 23c63e33d22e..972b66ea9554 100644--- a/products/alerts_platform/backend/logic/platform_reads.py+++ b/products/alerts_platform/backend/logic/platform_reads.py@@ -107,9 +107,10 @@ def list_configurations(limit: int,offset: int,legacy_configuration_id: UUID | None = None,+ configuration_ids: Collection[UUID] | None = None,) -> PlatformAlertConfigurationPage:"""One page of configurations, newest first, with the total behind it. With a- `legacy_configuration_id`, only the copy of that source row.+ `legacy_configuration_id`, only the copy of that source row. With `configuration_ids`, only those.The page is taken in the database rather than in the caller, so a team with many alertscosts one page rather than all of them. Slicing keeps the prefetch, which resolves over the@@ -118,6 +119,8 @@ def list_configurations(readable = _readable(team_id, source_kinds)if legacy_configuration_id is not None:readable = readable.filter(legacy_configuration_id=legacy_configuration_id)+ if configuration_ids is not None:+ readable = readable.filter(id__in=list(configuration_ids))return PlatformAlertConfigurationPage(configurations=tuple(_configuration_view(row) for row in readable[offset : offset + limit]),total=readable.count(),diff --git a/products/logs/backend/alert_signal_emitter.py b/products/logs/backend/alert_signal_emitter.pyindex 21fb65570bb5..c69e272da8d7 100644--- a/products/logs/backend/alert_signal_emitter.py+++ b/products/logs/backend/alert_signal_emitter.py@@ -2,15 +2,20 @@ from __future__ import annotationsimport dataclassesfrom typing import Literal+from uuid import UUIDfrom django.conf import settingsimport structlog+from posthog.dataclasses import frozenfrom posthog.models import Team+from products.alerts_platform.backend.facade.api import list_configurations+from products.alerts_platform.backend.facade.contracts import PlatformAlertConfigurationView, SourceKindfrom products.logs.backend.alert_state_machine import NotificationActionfrom products.logs.backend.models import LogsAlertConfiguration+from products.logs.backend.platform_source_config import filters_offrom products.signals.backend.facade.api import emit_signallogger = structlog.get_logger(__name__)@@ -153,33 +158,85 @@ async def emit_alert_state_change_signal(team: Team, na: NotifiedAlert) -> bool:return False+@frozen+class _AlertFacts:+ name: str+ threshold_count: int+ threshold_operator: str+ window_minutes: int+ filters: dict++ @classmethod+ def of_logs_alert(cls, alert: LogsAlertConfiguration) -> _AlertFacts:+ return cls(+ name=alert.name,+ threshold_count=alert.threshold_count,+ threshold_operator=alert.threshold_operator,+ window_minutes=alert.window_minutes,+ filters=alert.filters,+ )++ @classmethod+ def of_configuration(cls, configuration: PlatformAlertConfigurationView) -> _AlertFacts:+ condition = configuration.source_config["condition"]+ return cls(+ name=configuration.name,+ threshold_count=condition["threshold_count"],+ threshold_operator=condition["threshold_operator"],+ window_minutes=condition["window_minutes"],+ filters=filters_of(configuration.source_config),+ )+++def _facts_by_alert_id(team_id: int, alert_ids: set[str]) -> dict[str, _AlertFacts]:+ """What a signal says about each alert: from its logs row, or from the platform configuration+ when the platform created the alert without one."""+ facts = {+ str(alert.id): _AlertFacts.of_logs_alert(alert)+ for alert in LogsAlertConfiguration.objects.filter(team_id=team_id, id__in=alert_ids)+ }+ missing = [UUID(alert_id) for alert_id in alert_ids - facts.keys()]+ if missing:+ page = list_configurations(+ team_id=team_id,+ source_kinds=[SourceKind.LOGS.value],+ limit=len(missing),+ offset=0,+ configuration_ids=missing,+ )+ facts.update(+ {+ str(configuration.id): _AlertFacts.of_configuration(configuration)+ for configuration in page.configurations+ }+ )+ return facts++def notified_alerts_for_platform(team_id: int, signals: list[PlatformAlertSignal]) -> list[NotifiedAlert]:- """The signal facts for alerts the platform announced, read from their logs alert rows.+ """The signal facts for alerts the platform announced.- An alert whose logs row is gone gets no signal, the way the logs stack sends none for it.+ A copy whose logs row is gone gets no signal, the way the logs stack sends none for it."""- alerts = {- str(alert.id): alert- for alert in LogsAlertConfiguration.objects.filter(team_id=team_id, id__in=[s.alert_id for s in signals])- }+ facts_by_id = _facts_by_alert_id(team_id, {signal.alert_id for signal in signals})notified: list[NotifiedAlert] = []for signal in signals:- alert = alerts.get(signal.alert_id)- if alert is None:+ facts = facts_by_id.get(signal.alert_id)+ if facts is None:continuenotified.append(NotifiedAlert(alert_id=signal.alert_id,team_id=team_id,- alert_name=alert.name,+ alert_name=facts.name,action=signal.action,weight=_WEIGHT_BY_ACTION[signal.action],- threshold_count=alert.threshold_count,- threshold_operator=alert.threshold_operator,- window_minutes=alert.window_minutes,+ threshold_count=facts.threshold_count,+ threshold_operator=facts.threshold_operator,+ window_minutes=facts.window_minutes,result_count=signal.result_count,consecutive_failures=signal.consecutive_failures,- filters=alert.filters,+ filters=facts.filters,idempotency_key=signal.idempotency_key,group_labels=signal.labels,)diff --git a/products/logs/backend/alert_source_cycle.py b/products/logs/backend/alert_source_cycle.pyindex 81532408bed1..066f11770c08 100644--- a/products/logs/backend/alert_source_cycle.py+++ b/products/logs/backend/alert_source_cycle.py@@ -34,7 +34,13 @@ from posthog.models import Teamfrom posthog.temporal.alerts.admission import hold_evaluation_slot, release_evaluation_slotfrom products.alerts.backend.facade.destinations import configured_destination_template_ids-from products.alerts_platform.backend.facade.api import due_check_ids, due_checks, refresh_settings, slot_of+from products.alerts_platform.backend.facade.api import (+ due_check_ids,+ due_checks,+ has_incident_destination,+ refresh_settings,+ slot_of,+)from products.alerts_platform.backend.facade.contracts import (AlertDeliveryRequest,AlertEventKind,@@ -456,13 +462,13 @@ def _request(destination needs one resolve for every trigger. An alert with no such destination gets noincident action, so its edges start no delivery."""- destination_alert_id = str(check.legacy_configuration_id)+ destination_alert_id = str(check.destination_alert_id)actions = {key: actionfor key, move in moves.items()if (action := decide_incident_action(move.before, move.after, policy=PLATFORM_LOGS_ALERT_POLICY)) is not None}- if actions and not _has_incident_destination(check.team_id, destination_alert_id):+ if actions and not _has_incident_destination(check):actions = {}if not sends_messages and not actions:return None@@ -478,10 +484,9 @@ def _request(event_ids_by_incident_action=EVENT_IDS_BY_INCIDENT_ACTION,incident_actions=actions,sends_messages=sends_messages,- # The logs alert page, as the HogFunction messages linked. An alert with no logs row has none.- source_alert_url=logs_alert_url(check.team_id, check.legacy_configuration_id)- if check.legacy_configuration_id is not None- else None,+ # The logs alert page, as the HogFunction messages linked. The logs API serves an alert with+ # no logs row under the configuration's own id.+ source_alert_url=logs_alert_url(check.team_id, check.destination_alert_id),owned_by_platform=check.owned_by_platform,overflowed=recorded.overflowed,overflow_is_lower_bound=recorded.overflow_is_lower_bound,@@ -493,12 +498,18 @@ def _ungrouped_move(check: PlatformAlertCheckInput, outcome: Outcome) -> dict[streturn {"": _Move(before=AlertState(check.instance().state), after=outcome.new_state)}-def _has_incident_destination(team_id: int, destination_alert_id: str) -> bool:- """Whether a destination of this alert follows the incident events. Read only on a firing edge."""+def _has_incident_destination(check: PlatformAlertCheckInput) -> bool:+ """Whether a destination of this alert follows the incident events, as a HogFunction or as a+ destination the platform keeps. Read only on a firing edge. A copy's destinations are always+ HogFunctions, so only an alert with no logs row can have the platform's."""+ if check.legacy_configuration_id is None and has_incident_destination(+ team_id=check.team_id, configuration_id=check.id+ ):+ return Truereturn bool(configured_destination_template_ids(- team_id=team_id,- alert_id=destination_alert_id,+ team_id=check.team_id,+ alert_id=str(check.destination_alert_id),allowed_event_ids=tuple(EVENT_IDS_BY_INCIDENT_ACTION.values()),))@@ -678,6 +689,9 @@ def _as_configured_now(A copy the platform owns is not a seed: every edit reaches it directly, from the logs API or theplatform's, so it is evaluated as stored. It still needs its logs row on the copy's team. A row ina child environment has its logs there, and the query would read the project's instead.++ An alert the platform created without a logs row is evaluated as stored. It is always on a+ project, because a create refuses an environment."""legacy_ids = [check.legacy_configuration_id for check in checks if check.legacy_configuration_id is not None]alerts = {alert.id: alert for alert in LogsAlertConfiguration.objects.filter(team_id=team_id, id__in=legacy_ids)}@@ -685,7 +699,7 @@ def _as_configured_now(stopped: list[Decision] = []for check in checks:alert = alerts.get(check.legacy_configuration_id)- if check.owned_by_platform and alert is not None:+ if check.owned_by_platform and (check.legacy_configuration_id is None or alert is not None):current.append(check)continueif alert is None or not alert.enabled:diff --git a/products/logs/backend/test/test_alert_evaluate.py b/products/logs/backend/test/test_alert_evaluate.pyindex cb480705df6e..ce2e8db50dc7 100644--- a/products/logs/backend/test/test_alert_evaluate.py+++ b/products/logs/backend/test/test_alert_evaluate.py@@ -1,7 +1,7 @@import timefrom dataclasses import replacefrom datetime import UTC, datetime, timedelta-from typing import Any+from typing import Any, castfrom uuid import UUID, uuid4import pytest@@ -23,6 +23,7 @@ from posthog.temporal.alerts.admission import admit_evaluation_slotsfrom products.alerts_platform.backend.facade import testing as platform_testingfrom products.alerts_platform.backend.facade.api import (+ create_alert_destinations,due_checks,list_configurations,record_outcomes,@@ -31,12 +32,15 @@ from products.alerts_platform.backend.facade.api import ()from products.alerts_platform.backend.facade.contracts import (AlertDeliveryRequest,+ AlertDestinationData,AlertEventKind,CheckFailure,+ DestinationType,Grouping,GroupingMode,GroupOutcome,IncidentAction,+ NewAlertDestination,PlatformAlertOutcome,PlatformConfigurationSnapshot,SourceBatchEvaluation,@@ -76,6 +80,7 @@ from products.logs.backend.temporal.batch_slot import LogsBatchSlot_MODULE = "products.logs.backend.alert_source_cycle"CONDITION = {"threshold_count": 10, "threshold_operator": "above", "window_minutes": 5}+_NATIVE: dict[str, Any] = {"legacy_configuration_id": None, "destinations_owner": "platform"}_LOGS_OWNED_FIELDS = ("state", "consecutive_failures", "next_check_at", "last_notified_at", "snooze_until")# Settings a check reads from the logs alert rather than from its platform copy._LEGACY_ALERT_FIELDS = ("evaluation_periods", "datapoints_to_alarm", "cooldown_minutes", "schedule_restriction")@@ -114,7 +119,8 @@ class TestLogsAlertEvaluation(APIBaseTest):defaults.update(overrides)with team_scope(self.team.id):configuration = platform_testing.create_configuration(**defaults)- self._legacy_ids[configuration.id] = defaults["legacy_configuration_id"]+ if defaults["legacy_configuration_id"] is not None:+ self._legacy_ids[configuration.id] = defaults["legacy_configuration_id"]return configurationdef _run(@@ -143,6 +149,22 @@ class TestLogsAlertEvaluation(APIBaseTest):return evaluate_logs_batch(self.team.id, slot, now), querydef _add_incident_destination(self, configuration: PlatformConfigurationSnapshot) -> None:+ if configuration.id not in self._legacy_ids:+ with team_scope(self.team.id):+ create_alert_destinations(+ team_id=self.team.id,+ configuration_id=configuration.id,+ destinations=[+ NewAlertDestination(+ data=cast(+ AlertDestinationData,+ {"type": DestinationType.PAGERDUTY, "pagerduty_routing_key": "R-1"},+ ),+ event_kinds=(AlertEventKind.FIRING,),+ )+ ],+ )+ returnHogFunction.objects.create(team=self.team,type="internal_destination",@@ -177,15 +199,17 @@ class TestLogsAlertEvaluation(APIBaseTest):@parameterized.expand([- ("without_a_paging_destination", False, "source"),- ("with_a_paging_destination", True, "source"),- ("owned_by_the_platform", False, "platform"),+ ("without_a_paging_destination", False, "source", {}),+ ("with_a_paging_destination", True, "source", {}),+ ("owned_by_the_platform", False, "platform", {}),+ ("created_without_a_logs_row", False, "platform", _NATIVE),+ ("created_without_a_logs_row_and_paged_natively", True, "platform", _NATIVE),])def test_a_breaching_configuration_fires_and_records_its_own_state(- self, _name: str, paged: bool, owner: str+ self, _name: str, paged: bool, owner: str, fields: dict[str, Any]) -> None:- configuration = self._configuration(owner=owner)+ configuration = self._configuration(owner=owner, **fields)if paged:self._add_incident_destination(configuration)@@ -198,9 +222,11 @@ class TestLogsAlertEvaluation(APIBaseTest):]# The event a PagerDuty destination subscribes to, so the trigger reaches it.assert evaluation.deliveries[0].event_ids_by_incident_action["trigger"] == "$logs_alert_incident_opened"+ public_id = self._legacy_ids.get(configuration.id, configuration.id)+ assert evaluation.deliveries[0].destination_alert_id == str(public_id)source_alert_url = evaluation.deliveries[0].source_alert_urlassert source_alert_url is not None- assert source_alert_url.endswith(f"/logs/alerts/{self._legacy_ids[configuration.id]}")+ assert source_alert_url.endswith(f"/logs/alerts/{public_id}")with team_scope(self.team.id):alert = platform_testing.alert_for(configuration.id)assert alert is not None@@ -931,6 +957,15 @@ class TestPlatformAlertSignalFacts(APIBaseTest):alert = LogsAlertConfiguration.objects.create(team=self.team, name="API errors", filters={"serviceNames": ["api"]}, threshold_count=10, window_minutes=5)+ with team_scope(self.team.id):+ native = platform_testing.create_configuration(+ team_id=self.team.id,+ name="Web errors",+ source_kind=SourceKind.LOGS,+ source_config={"filters": {"serviceNames": ["web"]}, "condition": CONDITION},+ check_interval_minutes=10,+ owner="platform",+ )notified = notified_alerts_for_platform(self.team.id,@@ -954,16 +989,24 @@ class TestPlatformAlertSignalFacts(APIBaseTest):grouping_key='{"service_name":"web"}',labels={"service_name": "web"},),+ PlatformAlertSignal(+ alert_id=str(native.id),+ action="firing",+ result_count=30,+ consecutive_failures=0,+ evaluation_key="k-1",+ ),],)assert [(n.alert_id, n.alert_name, n.threshold_count, n.result_count, n.filters) for n in notified] == [(str(alert.id), "API errors", 10, 500, {"serviceNames": ["api"]}),(str(alert.id), "API errors", 10, 700, {"serviceNames": ["api"]}),+ (str(native.id), "Web errors", CONDITION["threshold_count"], 30, {"serviceNames": ["web"]}),]# A retried emission of the same announcement must not start a second signal, and a second# group of the same evaluation is not a retry.- assert [n.idempotency_key for n in notified] == [+ assert [n.idempotency_key for n in notified[:2]] == [f"{alert.id}:firing:k-1",f'{alert.id}:firing:k-1:{{"service_name":"web"}}',]