Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): evaluate platform logs alerts that have no logs row (#128696)

Jimmy Zhu145fc8ddc07a
diff --git a/products/alerts_platform/backend/facade/api.py b/products/alerts_platform/backend/facade/api.py
index 9132cbd60fd6..d1429e5fa1b0 100644
--- a/products/alerts_platform/backend/facade/api.py
+++ b/products/alerts_platform/backend/facade/api.py
@@ -189,15 +189,17 @@ def list_configurations(
limit: int,
offset: int,
legacy_configuration_id: UUID | None = None,
+ configuration_ids: Collection[UUID] | None = None,
) -> PlatformAlertConfigurationPage:
"""One page of configurations a caller may read, with the total behind it. With a
- `legacy_configuration_id`, only the copy of that source row."""
+ `legacy_configuration_id`, only the copy of that source row. With `configuration_ids`, only those."""
return platform_reads.list_configurations(
team_id=team_id,
source_kinds=source_kinds,
limit=limit,
offset=offset,
legacy_configuration_id=legacy_configuration_id,
+ configuration_ids=configuration_ids,
)
diff --git a/products/alerts_platform/backend/logic/platform_reads.py b/products/alerts_platform/backend/logic/platform_reads.py
index 23c63e33d22e..972b66ea9554 100644
--- a/products/alerts_platform/backend/logic/platform_reads.py
+++ b/products/alerts_platform/backend/logic/platform_reads.py
@@ -107,9 +107,10 @@ def list_configurations(
limit: int,
offset: int,
legacy_configuration_id: UUID | None = None,
+ configuration_ids: Collection[UUID] | None = None,
) -> PlatformAlertConfigurationPage:
"""One page of configurations, newest first, with the total behind it. With a
- `legacy_configuration_id`, only the copy of that source row.
+ `legacy_configuration_id`, only the copy of that source row. With `configuration_ids`, only those.
The page is taken in the database rather than in the caller, so a team with many alerts
costs one page rather than all of them. Slicing keeps the prefetch, which resolves over the
@@ -118,6 +119,8 @@ def list_configurations(
readable = _readable(team_id, source_kinds)
if legacy_configuration_id is not None:
readable = readable.filter(legacy_configuration_id=legacy_configuration_id)
+ if configuration_ids is not None:
+ readable = readable.filter(id__in=list(configuration_ids))
return PlatformAlertConfigurationPage(
configurations=tuple(_configuration_view(row) for row in readable[offset : offset + limit]),
total=readable.count(),
diff --git a/products/logs/backend/alert_signal_emitter.py b/products/logs/backend/alert_signal_emitter.py
index 21fb65570bb5..c69e272da8d7 100644
--- a/products/logs/backend/alert_signal_emitter.py
+++ b/products/logs/backend/alert_signal_emitter.py
@@ -2,15 +2,20 @@ from __future__ import annotations
import dataclasses
from typing import Literal
+from uuid import UUID
from django.conf import settings
import structlog
+from posthog.dataclasses import frozen
from posthog.models import Team
+from products.alerts_platform.backend.facade.api import list_configurations
+from products.alerts_platform.backend.facade.contracts import PlatformAlertConfigurationView, SourceKind
from products.logs.backend.alert_state_machine import NotificationAction
from products.logs.backend.models import LogsAlertConfiguration
+from products.logs.backend.platform_source_config import filters_of
from products.signals.backend.facade.api import emit_signal
logger = structlog.get_logger(__name__)
@@ -153,33 +158,85 @@ async def emit_alert_state_change_signal(team: Team, na: NotifiedAlert) -> bool:
return False
+@frozen
+class _AlertFacts:
+ name: str
+ threshold_count: int
+ threshold_operator: str
+ window_minutes: int
+ filters: dict
+
+ @classmethod
+ def of_logs_alert(cls, alert: LogsAlertConfiguration) -> _AlertFacts:
+ return cls(
+ name=alert.name,
+ threshold_count=alert.threshold_count,
+ threshold_operator=alert.threshold_operator,
+ window_minutes=alert.window_minutes,
+ filters=alert.filters,
+ )
+
+ @classmethod
+ def of_configuration(cls, configuration: PlatformAlertConfigurationView) -> _AlertFacts:
+ condition = configuration.source_config["condition"]
+ return cls(
+ name=configuration.name,
+ threshold_count=condition["threshold_count"],
+ threshold_operator=condition["threshold_operator"],
+ window_minutes=condition["window_minutes"],
+ filters=filters_of(configuration.source_config),
+ )
+
+
+def _facts_by_alert_id(team_id: int, alert_ids: set[str]) -> dict[str, _AlertFacts]:
+ """What a signal says about each alert: from its logs row, or from the platform configuration
+ when the platform created the alert without one."""
+ facts = {
+ str(alert.id): _AlertFacts.of_logs_alert(alert)
+ for alert in LogsAlertConfiguration.objects.filter(team_id=team_id, id__in=alert_ids)
+ }
+ missing = [UUID(alert_id) for alert_id in alert_ids - facts.keys()]
+ if missing:
+ page = list_configurations(
+ team_id=team_id,
+ source_kinds=[SourceKind.LOGS.value],
+ limit=len(missing),
+ offset=0,
+ configuration_ids=missing,
+ )
+ facts.update(
+ {
+ str(configuration.id): _AlertFacts.of_configuration(configuration)
+ for configuration in page.configurations
+ }
+ )
+ return facts
+
+
def notified_alerts_for_platform(team_id: int, signals: list[PlatformAlertSignal]) -> list[NotifiedAlert]:
- """The signal facts for alerts the platform announced, read from their logs alert rows.
+ """The signal facts for alerts the platform announced.
- An alert whose logs row is gone gets no signal, the way the logs stack sends none for it.
+ A copy whose logs row is gone gets no signal, the way the logs stack sends none for it.
"""
- alerts = {
- str(alert.id): alert
- for alert in LogsAlertConfiguration.objects.filter(team_id=team_id, id__in=[s.alert_id for s in signals])
- }
+ facts_by_id = _facts_by_alert_id(team_id, {signal.alert_id for signal in signals})
notified: list[NotifiedAlert] = []
for signal in signals:
- alert = alerts.get(signal.alert_id)
- if alert is None:
+ facts = facts_by_id.get(signal.alert_id)
+ if facts is None:
continue
notified.append(
NotifiedAlert(
alert_id=signal.alert_id,
team_id=team_id,
- alert_name=alert.name,
+ alert_name=facts.name,
action=signal.action,
weight=_WEIGHT_BY_ACTION[signal.action],
- threshold_count=alert.threshold_count,
- threshold_operator=alert.threshold_operator,
- window_minutes=alert.window_minutes,
+ threshold_count=facts.threshold_count,
+ threshold_operator=facts.threshold_operator,
+ window_minutes=facts.window_minutes,
result_count=signal.result_count,
consecutive_failures=signal.consecutive_failures,
- filters=alert.filters,
+ filters=facts.filters,
idempotency_key=signal.idempotency_key,
group_labels=signal.labels,
)
diff --git a/products/logs/backend/alert_source_cycle.py b/products/logs/backend/alert_source_cycle.py
index 81532408bed1..066f11770c08 100644
--- a/products/logs/backend/alert_source_cycle.py
+++ b/products/logs/backend/alert_source_cycle.py
@@ -34,7 +34,13 @@ from posthog.models import Team
from posthog.temporal.alerts.admission import hold_evaluation_slot, release_evaluation_slot
from products.alerts.backend.facade.destinations import configured_destination_template_ids
-from products.alerts_platform.backend.facade.api import due_check_ids, due_checks, refresh_settings, slot_of
+from products.alerts_platform.backend.facade.api import (
+ due_check_ids,
+ due_checks,
+ has_incident_destination,
+ refresh_settings,
+ slot_of,
+)
from products.alerts_platform.backend.facade.contracts import (
AlertDeliveryRequest,
AlertEventKind,
@@ -456,13 +462,13 @@ def _request(
destination needs one resolve for every trigger. An alert with no such destination gets no
incident action, so its edges start no delivery.
"""
- destination_alert_id = str(check.legacy_configuration_id)
+ destination_alert_id = str(check.destination_alert_id)
actions = {
key: action
for key, move in moves.items()
if (action := decide_incident_action(move.before, move.after, policy=PLATFORM_LOGS_ALERT_POLICY)) is not None
}
- if actions and not _has_incident_destination(check.team_id, destination_alert_id):
+ if actions and not _has_incident_destination(check):
actions = {}
if not sends_messages and not actions:
return None
@@ -478,10 +484,9 @@ def _request(
event_ids_by_incident_action=EVENT_IDS_BY_INCIDENT_ACTION,
incident_actions=actions,
sends_messages=sends_messages,
- # The logs alert page, as the HogFunction messages linked. An alert with no logs row has none.
- source_alert_url=logs_alert_url(check.team_id, check.legacy_configuration_id)
- if check.legacy_configuration_id is not None
- else None,
+ # The logs alert page, as the HogFunction messages linked. The logs API serves an alert with
+ # no logs row under the configuration's own id.
+ source_alert_url=logs_alert_url(check.team_id, check.destination_alert_id),
owned_by_platform=check.owned_by_platform,
overflowed=recorded.overflowed,
overflow_is_lower_bound=recorded.overflow_is_lower_bound,
@@ -493,12 +498,18 @@ def _ungrouped_move(check: PlatformAlertCheckInput, outcome: Outcome) -> dict[st
return {"": _Move(before=AlertState(check.instance().state), after=outcome.new_state)}
-def _has_incident_destination(team_id: int, destination_alert_id: str) -> bool:
- """Whether a destination of this alert follows the incident events. Read only on a firing edge."""
+def _has_incident_destination(check: PlatformAlertCheckInput) -> bool:
+ """Whether a destination of this alert follows the incident events, as a HogFunction or as a
+ destination the platform keeps. Read only on a firing edge. A copy's destinations are always
+ HogFunctions, so only an alert with no logs row can have the platform's."""
+ if check.legacy_configuration_id is None and has_incident_destination(
+ team_id=check.team_id, configuration_id=check.id
+ ):
+ return True
return bool(
configured_destination_template_ids(
- team_id=team_id,
- alert_id=destination_alert_id,
+ team_id=check.team_id,
+ alert_id=str(check.destination_alert_id),
allowed_event_ids=tuple(EVENT_IDS_BY_INCIDENT_ACTION.values()),
)
)
@@ -678,6 +689,9 @@ def _as_configured_now(
A copy the platform owns is not a seed: every edit reaches it directly, from the logs API or the
platform's, so it is evaluated as stored. It still needs its logs row on the copy's team. A row in
a child environment has its logs there, and the query would read the project's instead.
+
+ An alert the platform created without a logs row is evaluated as stored. It is always on a
+ project, because a create refuses an environment.
"""
legacy_ids = [check.legacy_configuration_id for check in checks if check.legacy_configuration_id is not None]
alerts = {alert.id: alert for alert in LogsAlertConfiguration.objects.filter(team_id=team_id, id__in=legacy_ids)}
@@ -685,7 +699,7 @@ def _as_configured_now(
stopped: list[Decision] = []
for check in checks:
alert = alerts.get(check.legacy_configuration_id)
- if check.owned_by_platform and alert is not None:
+ if check.owned_by_platform and (check.legacy_configuration_id is None or alert is not None):
current.append(check)
continue
if alert is None or not alert.enabled:
diff --git a/products/logs/backend/test/test_alert_evaluate.py b/products/logs/backend/test/test_alert_evaluate.py
index cb480705df6e..ce2e8db50dc7 100644
--- a/products/logs/backend/test/test_alert_evaluate.py
+++ b/products/logs/backend/test/test_alert_evaluate.py
@@ -1,7 +1,7 @@
import time
from dataclasses import replace
from datetime import UTC, datetime, timedelta
-from typing import Any
+from typing import Any, cast
from uuid import UUID, uuid4
import pytest
@@ -23,6 +23,7 @@ from posthog.temporal.alerts.admission import admit_evaluation_slots
from products.alerts_platform.backend.facade import testing as platform_testing
from products.alerts_platform.backend.facade.api import (
+ create_alert_destinations,
due_checks,
list_configurations,
record_outcomes,
@@ -31,12 +32,15 @@ from products.alerts_platform.backend.facade.api import (
)
from products.alerts_platform.backend.facade.contracts import (
AlertDeliveryRequest,
+ AlertDestinationData,
AlertEventKind,
CheckFailure,
+ DestinationType,
Grouping,
GroupingMode,
GroupOutcome,
IncidentAction,
+ NewAlertDestination,
PlatformAlertOutcome,
PlatformConfigurationSnapshot,
SourceBatchEvaluation,
@@ -76,6 +80,7 @@ from products.logs.backend.temporal.batch_slot import LogsBatchSlot
_MODULE = "products.logs.backend.alert_source_cycle"
CONDITION = {"threshold_count": 10, "threshold_operator": "above", "window_minutes": 5}
+_NATIVE: dict[str, Any] = {"legacy_configuration_id": None, "destinations_owner": "platform"}
_LOGS_OWNED_FIELDS = ("state", "consecutive_failures", "next_check_at", "last_notified_at", "snooze_until")
# Settings a check reads from the logs alert rather than from its platform copy.
_LEGACY_ALERT_FIELDS = ("evaluation_periods", "datapoints_to_alarm", "cooldown_minutes", "schedule_restriction")
@@ -114,7 +119,8 @@ class TestLogsAlertEvaluation(APIBaseTest):
defaults.update(overrides)
with team_scope(self.team.id):
configuration = platform_testing.create_configuration(**defaults)
- self._legacy_ids[configuration.id] = defaults["legacy_configuration_id"]
+ if defaults["legacy_configuration_id"] is not None:
+ self._legacy_ids[configuration.id] = defaults["legacy_configuration_id"]
return configuration
def _run(
@@ -143,6 +149,22 @@ class TestLogsAlertEvaluation(APIBaseTest):
return evaluate_logs_batch(self.team.id, slot, now), query
def _add_incident_destination(self, configuration: PlatformConfigurationSnapshot) -> None:
+ if configuration.id not in self._legacy_ids:
+ with team_scope(self.team.id):
+ create_alert_destinations(
+ team_id=self.team.id,
+ configuration_id=configuration.id,
+ destinations=[
+ NewAlertDestination(
+ data=cast(
+ AlertDestinationData,
+ {"type": DestinationType.PAGERDUTY, "pagerduty_routing_key": "R-1"},
+ ),
+ event_kinds=(AlertEventKind.FIRING,),
+ )
+ ],
+ )
+ return
HogFunction.objects.create(
team=self.team,
type="internal_destination",
@@ -177,15 +199,17 @@ class TestLogsAlertEvaluation(APIBaseTest):
@parameterized.expand(
[
- ("without_a_paging_destination", False, "source"),
- ("with_a_paging_destination", True, "source"),
- ("owned_by_the_platform", False, "platform"),
+ ("without_a_paging_destination", False, "source", {}),
+ ("with_a_paging_destination", True, "source", {}),
+ ("owned_by_the_platform", False, "platform", {}),
+ ("created_without_a_logs_row", False, "platform", _NATIVE),
+ ("created_without_a_logs_row_and_paged_natively", True, "platform", _NATIVE),
]
)
def test_a_breaching_configuration_fires_and_records_its_own_state(
- self, _name: str, paged: bool, owner: str
+ self, _name: str, paged: bool, owner: str, fields: dict[str, Any]
) -> None:
- configuration = self._configuration(owner=owner)
+ configuration = self._configuration(owner=owner, **fields)
if paged:
self._add_incident_destination(configuration)
@@ -198,9 +222,11 @@ class TestLogsAlertEvaluation(APIBaseTest):
]
# The event a PagerDuty destination subscribes to, so the trigger reaches it.
assert evaluation.deliveries[0].event_ids_by_incident_action["trigger"] == "$logs_alert_incident_opened"
+ public_id = self._legacy_ids.get(configuration.id, configuration.id)
+ assert evaluation.deliveries[0].destination_alert_id == str(public_id)
source_alert_url = evaluation.deliveries[0].source_alert_url
assert source_alert_url is not None
- assert source_alert_url.endswith(f"/logs/alerts/{self._legacy_ids[configuration.id]}")
+ assert source_alert_url.endswith(f"/logs/alerts/{public_id}")
with team_scope(self.team.id):
alert = platform_testing.alert_for(configuration.id)
assert alert is not None
@@ -931,6 +957,15 @@ class TestPlatformAlertSignalFacts(APIBaseTest):
alert = LogsAlertConfiguration.objects.create(
team=self.team, name="API errors", filters={"serviceNames": ["api"]}, threshold_count=10, window_minutes=5
)
+ with team_scope(self.team.id):
+ native = platform_testing.create_configuration(
+ team_id=self.team.id,
+ name="Web errors",
+ source_kind=SourceKind.LOGS,
+ source_config={"filters": {"serviceNames": ["web"]}, "condition": CONDITION},
+ check_interval_minutes=10,
+ owner="platform",
+ )
notified = notified_alerts_for_platform(
self.team.id,
@@ -954,16 +989,24 @@ class TestPlatformAlertSignalFacts(APIBaseTest):
grouping_key='{"service_name":"web"}',
labels={"service_name": "web"},
),
+ PlatformAlertSignal(
+ alert_id=str(native.id),
+ action="firing",
+ result_count=30,
+ consecutive_failures=0,
+ evaluation_key="k-1",
+ ),
],
)
assert [(n.alert_id, n.alert_name, n.threshold_count, n.result_count, n.filters) for n in notified] == [
(str(alert.id), "API errors", 10, 500, {"serviceNames": ["api"]}),
(str(alert.id), "API errors", 10, 700, {"serviceNames": ["api"]}),
+ (str(native.id), "Web errors", CONDITION["threshold_count"], 30, {"serviceNames": ["web"]}),
]
# A retried emission of the same announcement must not start a second signal, and a second
# group of the same evaluation is not a retry.
- assert [n.idempotency_key for n in notified] == [
+ assert [n.idempotency_key for n in notified[:2]] == [
f"{alert.id}:firing:k-1",
f'{alert.id}:firing:k-1:{{"service_name":"web"}}',
]