Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(alerts): count deliveries, delivery lag and alert ownership (#122944)

Jimmy Zhu9ebb4dce857f
diff --git a/products/alerts_platform/backend/delivery/dispatch.py b/products/alerts_platform/backend/delivery/dispatch.py
index 33768cfc6902..1052318a2a5f 100644
--- a/products/alerts_platform/backend/delivery/dispatch.py
+++ b/products/alerts_platform/backend/delivery/dispatch.py
@@ -5,6 +5,8 @@ it. Which destinations an alert has is resolved before this, so a change to that
reaches no transport.
"""
+from datetime import UTC
+
from django.utils import timezone
import structlog
@@ -22,12 +24,18 @@ from products.alerts_platform.backend.delivery.transport import (
from products.alerts_platform.backend.facade.contracts import (
AlertDestinationData,
AnnouncedTransition,
+ DeliveryPath,
EvaluationAnnouncement,
IncidentAction,
MessageLink,
)
from products.alerts_platform.backend.logic.platform_reads import alert_snapshot
-from products.alerts_platform.backend.temporal.metrics import increment_root_edit_failures, safe_record
+from products.alerts_platform.backend.temporal.metrics import (
+ increment_deliveries,
+ increment_root_edit_failures,
+ record_delivery_lag,
+ safe_record,
+)
logger = structlog.get_logger(__name__)
@@ -48,6 +56,7 @@ def deliver(
project: MessageLink | None = None,
) -> None:
channel_target = transport.channel_target(target)
+ source = announcement.source.value
busy = 0
for transition in announcement.transitions:
@@ -67,8 +76,12 @@ def deliver(
# in the channel twice. The holder may also have died, and then its claim frees
# this thread only when it goes stale. The caller waits and runs the send again.
busy += 1
+ safe_record(increment_deliveries, source, transport.provider, DeliveryPath.NATIVE.value, "busy")
continue
if claim is None:
+ safe_record(
+ increment_deliveries, source, transport.provider, DeliveryPath.NATIVE.value, "already_delivered"
+ )
continue
try:
@@ -128,6 +141,7 @@ def _send(
succeeded=False,
error=str(error) if isinstance(error, DeliveryError) else type(error).__name__,
)
+ safe_record(increment_deliveries, message.source.value, transport.provider, DeliveryPath.NATIVE.value, "failed")
raise
record_delivery(
team_id=team_id,
@@ -136,9 +150,19 @@ def _send(
destination_id=destination_id,
succeeded=True,
)
+ safe_record(increment_deliveries, message.source.value, transport.provider, DeliveryPath.NATIVE.value, "sent")
+ safe_record(record_delivery_lag, message.source.value, transport.provider, _lag_ms(message.transition))
return sent
+def _lag_ms(transition: AnnouncedTransition) -> int:
+ # History rows come back from ClickHouse without a zone, and their times are UTC.
+ decided_at = transition.occurred_at
+ if decided_at.tzinfo is None:
+ decided_at = decided_at.replace(tzinfo=UTC)
+ return max(0, int((timezone.now() - decided_at).total_seconds() * 1000))
+
+
def _edit_root(
*,
transport: DeliveryTransport,
diff --git a/products/alerts_platform/backend/delivery/evaluation.py b/products/alerts_platform/backend/delivery/evaluation.py
index 8ec95d3bf56a..31fa353332e0 100644
--- a/products/alerts_platform/backend/delivery/evaluation.py
+++ b/products/alerts_platform/backend/delivery/evaluation.py
@@ -45,6 +45,7 @@ from products.alerts_platform.backend.facade.contracts import (
MessageLink,
)
from products.alerts_platform.backend.logic.platform_alert_events import announced_kinds, announcement
+from products.alerts_platform.backend.temporal.metrics import DeliveryOutcomeLabel, increment_deliveries, safe_record
logger = structlog.get_logger(__name__)
@@ -223,6 +224,14 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
transport_class = _TRANSPORTS.get(target["type"])
if transport_class is None:
skipped += 1
+ safe_record(
+ increment_deliveries,
+ request.source.value,
+ target["type"],
+ DeliveryPath.NATIVE.value,
+ "no_transport",
+ len(transitions),
+ )
continue
try:
transport = webhook_transports.get() if transport_class is WebhookTransport else transport_class()
@@ -237,6 +246,14 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
succeeded=False,
error=str(error),
)
+ safe_record(
+ increment_deliveries,
+ request.source.value,
+ WebhookTransport.provider,
+ DeliveryPath.NATIVE.value,
+ "failed",
+ len(transitions),
+ )
failures.append(str(error))
first_refusal = first_refusal or error
continue
@@ -294,6 +311,10 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
except ThreadBusy as error:
busy.append(str(error))
emission = None
+ _count_emissions(request, "busy", len(emissions))
+ else:
+ if emission is None:
+ _count_emissions(request, "already_delivered", len(emissions))
if emission is not None:
try:
emitter(request=request, announcement=announced, emissions=emissions)
@@ -306,9 +327,11 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
exc_info=True,
)
failures.append(f"legacy events: {type(error).__name__}")
+ _count_emissions(request, "failed", len(emissions))
else:
thread_store.delivered(emission, None)
emitted = emissions
+ _count_emissions(request, "sent", len(emissions))
# A held thread wins over a failure. The workflow waits a held thread out and then runs the
# whole delivery again, which retries the failed destinations too. A failure raised instead
# spends the activity's retries inside the claim's TTL, and the held message is lost.
@@ -326,6 +349,18 @@ def deliver_evaluation(request: AlertDeliveryRequest) -> DeliveryOutcome:
)
+def _count_emissions(request: AlertDeliveryRequest, outcome: DeliveryOutcomeLabel, transitions: int) -> None:
+ # The alert's own HogFunctions send these, so the provider is the path itself.
+ safe_record(
+ increment_deliveries,
+ request.source.value,
+ DeliveryPath.HOG_FUNCTION.value,
+ DeliveryPath.HOG_FUNCTION.value,
+ outcome,
+ transitions,
+ )
+
+
def _split_by_path(
request: AlertDeliveryRequest, announced: EvaluationAnnouncement
) -> tuple[EvaluationAnnouncement, tuple[LegacyEventEmission, ...]]:
diff --git a/products/alerts_platform/backend/logic/inventory.py b/products/alerts_platform/backend/logic/inventory.py
index 96300c804aa6..4728279c0994 100644
--- a/products/alerts_platform/backend/logic/inventory.py
+++ b/products/alerts_platform/backend/logic/inventory.py
@@ -12,7 +12,7 @@ from posthog.dataclasses import frozen
from posthog.models.utils import execute_with_timeout
from products.alerts_platform.backend.facade.contracts import SourceKind
-from products.alerts_platform.backend.facade.enums import PlatformAlertState
+from products.alerts_platform.backend.facade.enums import PlatformAlertOwner, PlatformAlertState
from products.alerts_platform.backend.models import PlatformAlert, PlatformAlertConfiguration
# Per statement. The counts scan every configuration, unlike discovery, which reads only the due index,
@@ -20,6 +20,14 @@ from products.alerts_platform.backend.models import PlatformAlert, PlatformAlert
INVENTORY_STATEMENT_TIMEOUT_MS = 500
+@frozen
+class ConfigurationCount:
+ enabled: bool
+ # Which stack decides and sends for the alert, so the count shows how far a cutover has gone.
+ owner: PlatformAlertOwner
+ count: int
+
+
@frozen
class AlertCount:
state: PlatformAlertState
@@ -30,8 +38,7 @@ class AlertCount:
@frozen
class SourceInventory:
source: SourceKind
- enabled_configurations: int
- disabled_configurations: int
+ configurations: tuple[ConfigurationCount, ...]
alerts: tuple[AlertCount, ...]
@@ -40,9 +47,9 @@ def count_inventory(now: dt.datetime) -> tuple[SourceInventory, ...]:
# because the timeout is transaction-local.
with execute_with_timeout(INVENTORY_STATEMENT_TIMEOUT_MS):
configuration_counts = {
- (row["source_kind"], row["enabled"]): row["n"]
+ (row["source_kind"], row["enabled"], row["owner"]): row["n"]
for row in PlatformAlertConfiguration.objects.unscoped()
- .values("source_kind", "enabled")
+ .values("source_kind", "enabled", "owner")
.annotate(n=Count("id"))
}
is_muted = Q(snooze_until__gt=now) | Q(configuration__snooze_until__gt=now)
@@ -64,8 +71,15 @@ def count_inventory(now: dt.datetime) -> tuple[SourceInventory, ...]:
inventories.append(
SourceInventory(
source=source,
- enabled_configurations=configuration_counts.get((source.value, True), 0),
- disabled_configurations=configuration_counts.get((source.value, False), 0),
+ configurations=tuple(
+ ConfigurationCount(
+ enabled=enabled,
+ owner=owner,
+ count=configuration_counts.get((source.value, enabled, owner.value), 0),
+ )
+ for enabled in (True, False)
+ for owner in PlatformAlertOwner
+ ),
alerts=tuple(alerts),
)
)
diff --git a/products/alerts_platform/backend/temporal/inventory_metrics.py b/products/alerts_platform/backend/temporal/inventory_metrics.py
index 6bdb9e658bdd..2f034b34ca04 100644
--- a/products/alerts_platform/backend/temporal/inventory_metrics.py
+++ b/products/alerts_platform/backend/temporal/inventory_metrics.py
@@ -22,8 +22,8 @@ def push_inventory(inventories: Sequence[SourceInventory]) -> None:
with pushed_metrics_registry("alerts_platform_inventory", timeout=PUSH_TIMEOUT_SECONDS) as registry:
configurations = Gauge(
"alerts_platform_configurations",
- "Platform alert configurations, by source and whether they are enabled",
- ["source", "enabled"],
+ "Platform alert configurations, by source, whether they are enabled, and which stack owns them",
+ ["source", "enabled", "owner"],
registry=registry,
)
alerts = Gauge(
@@ -34,7 +34,9 @@ def push_inventory(inventories: Sequence[SourceInventory]) -> None:
)
for inventory in inventories:
source = inventory.source.value
- configurations.labels(source=source, enabled="true").set(inventory.enabled_configurations)
- configurations.labels(source=source, enabled="false").set(inventory.disabled_configurations)
+ for configuration in inventory.configurations:
+ configurations.labels(
+ source=source, enabled=str(configuration.enabled).lower(), owner=configuration.owner.value
+ ).set(configuration.count)
for alert in inventory.alerts:
alerts.labels(source=source, state=alert.state.value, muted=str(alert.muted).lower()).set(alert.count)
diff --git a/products/alerts_platform/backend/temporal/metrics.py b/products/alerts_platform/backend/temporal/metrics.py
index c6a05c1934e4..ecc850bede78 100644
--- a/products/alerts_platform/backend/temporal/metrics.py
+++ b/products/alerts_platform/backend/temporal/metrics.py
@@ -7,9 +7,13 @@ below. Every call site wraps these in `safe_record`: a metric must never fail a
import datetime as dt
from collections.abc import Callable
+from functools import cache
+from typing import Literal
+
+from prometheus_client import Counter, Histogram
from posthog.temporal.common.logger import get_write_only_logger
-from posthog.temporal.common.metrics import get_metric_meter
+from posthog.temporal.common.metrics import Attributes, get_metric_meter
logger = get_write_only_logger(__name__)
@@ -18,6 +22,7 @@ logger = get_write_only_logger(__name__)
ALERTS_PLATFORM_LATENCY_HISTOGRAM_METRICS = (
"alerts_platform_batch_duration_ms",
"alerts_platform_scheduler_lag_ms",
+ "alerts_platform_delivery_lag_ms",
)
ALERTS_PLATFORM_LATENCY_HISTOGRAM_BUCKETS = [
@@ -194,6 +199,61 @@ def record_batch_duration(source: str, duration_ms: int) -> None:
).record(dt.timedelta(milliseconds=duration_ms))
+DeliveryOutcomeLabel = Literal["sent", "failed", "already_delivered", "busy", "no_transport"]
+
+_DELIVERIES_DESCRIPTION = "Transitions a delivery handled, by provider, delivery path and outcome"
+_DELIVERY_LAG_DESCRIPTION = "Delay between the check that decided a transition and the send that announced it"
+
+
+# A web process sends inline, outside any worker, when Temporal is unreachable or a deleted alert's
+# resolve cannot wait. It has no Temporal meter, so it records the same series in its own registry.
+# Created on first use, never at import: a worker exports this registry beside Temporal's meter, and
+# the same family declared twice makes Prometheus reject the worker's whole scrape.
+@cache
+def _inline_deliveries() -> Counter:
+ return Counter("alerts_platform_deliveries", _DELIVERIES_DESCRIPTION, ["source", "provider", "path", "outcome"])
+
+
+@cache
+def _inline_delivery_lag() -> Histogram:
+ return Histogram(
+ "alerts_platform_delivery_lag_ms",
+ _DELIVERY_LAG_DESCRIPTION,
+ ["source", "provider"],
+ buckets=ALERTS_PLATFORM_LATENCY_HISTOGRAM_BUCKETS,
+ )
+
+
+def increment_deliveries(source: str, provider: str, path: str, outcome: DeliveryOutcomeLabel, count: int = 1) -> None:
+ """What happened to each transition a delivery carried, per destination provider.
+
+ `already_delivered` is a retry finding its send done, and `busy` a send that waits on another
+ one holding the thread. Neither sent anything, so a rising `sent` with a flat `failed` is the
+ only reading that says alerts reach people.
+ """
+ labels: Attributes = {"source": source, "provider": provider, "path": path, "outcome": outcome}
+ try:
+ meter = get_metric_meter(labels)
+ except RuntimeError:
+ _inline_deliveries().labels(**labels).inc(count)
+ return
+ meter.create_counter("alerts_platform_deliveries_total", _DELIVERIES_DESCRIPTION).add(count)
+
+
+def record_delivery_lag(source: str, provider: str, lag_ms: int) -> None:
+ """How long after the check decided a transition its message went out. Evaluation metrics
+ cannot see a delivery queue that stalls, so this is the one that does."""
+ labels: Attributes = {"source": source, "provider": provider}
+ try:
+ meter = get_metric_meter(labels)
+ except RuntimeError:
+ _inline_delivery_lag().labels(**labels).observe(lag_ms)
+ return
+ meter.create_histogram_timedelta(
+ name="alerts_platform_delivery_lag_ms", description=_DELIVERY_LAG_DESCRIPTION, unit="ms"
+ ).record(dt.timedelta(milliseconds=lag_ms))
+
+
def record_scheduler_lag(source: str, lag_ms: int) -> None:
"""How far past its due time a check was evaluated. Rising lag means discovery's bound is
biting, because a key it omits grows more overdue until a later tick takes it."""
diff --git a/products/alerts_platform/backend/tests/recorded_metrics.py b/products/alerts_platform/backend/tests/recorded_metrics.py
new file mode 100644
index 000000000000..b898514500f9
--- /dev/null
+++ b/products/alerts_platform/backend/tests/recorded_metrics.py
@@ -0,0 +1,52 @@
+from collections.abc import Iterator
+from contextlib import contextmanager
+from datetime import timedelta
+
+from unittest.mock import patch
+
+from posthog.dataclasses import frozen
+
+
+@frozen
+class RecordedMetric:
+ name: str
+ attributes: dict[str, str]
+ value: float
+
+
+class _Instrument:
+ def __init__(self, name: str, attributes: dict[str, str], records: list[RecordedMetric]) -> None:
+ self._name = name
+ self._attributes = attributes
+ self._records = records
+
+ def add(self, value: int) -> None:
+ self._records.append(RecordedMetric(name=self._name, attributes=self._attributes, value=value))
+
+ def record(self, value: timedelta) -> None:
+ self._records.append(
+ RecordedMetric(name=self._name, attributes=self._attributes, value=value.total_seconds() * 1000)
+ )
+
+
+class _Meter:
+ def __init__(self, attributes: dict[str, str] | None, records: list[RecordedMetric]) -> None:
+ self._attributes = attributes or {}
+ self._records = records
+
+ def create_counter(self, name: str, *_args: object) -> _Instrument:
+ return _Instrument(name, self._attributes, self._records)
+
+ def create_histogram_timedelta(self, name: str, **_kwargs: object) -> _Instrument:
+ return _Instrument(name, self._attributes, self._records)
+
+
+@contextmanager
+def recorded_metrics() -> Iterator[list[RecordedMetric]]:
+ """Every platform metric recorded inside the block, captured where it reaches Temporal's meter."""
+ records: list[RecordedMetric] = []
+ with patch(
+ "products.alerts_platform.backend.temporal.metrics.get_metric_meter",
+ side_effect=lambda attributes=None: _Meter(attributes, records),
+ ):
+ yield records
diff --git a/products/alerts_platform/backend/tests/test_delivery_dispatch.py b/products/alerts_platform/backend/tests/test_delivery_dispatch.py
index 6483dbff2991..5565be424256 100644
--- a/products/alerts_platform/backend/tests/test_delivery_dispatch.py
+++ b/products/alerts_platform/backend/tests/test_delivery_dispatch.py
@@ -9,6 +9,7 @@ from django.test import SimpleTestCase
from django.utils import timezone
from parameterized import parameterized
+from prometheus_client import REGISTRY
from products.alerts_platform.backend.delivery.dispatch import deliver
from products.alerts_platform.backend.delivery.message import AlertMessage
@@ -22,6 +23,7 @@ from products.alerts_platform.backend.facade.contracts import (
EvaluationAnnouncement,
SourceKind,
)
+from products.alerts_platform.backend.tests.recorded_metrics import recorded_metrics
TARGET = cast(AlertDestinationData, {"type": "slack", "slack_workspace_id": 1, "slack_channel_id": "C-ENG"})
FIRST_FIRING = datetime(2026, 9, 30, 9, tzinfo=UTC)
@@ -249,12 +251,19 @@ class TestDeliveryDispatch(SimpleTestCase):
def test_a_retried_evaluation_does_not_send_a_second_copy(self) -> None:
store = RecordingThreadStore()
- self._deliver(FakeTransport(), store, evaluation_key="eval-1")
-
retry = FakeTransport()
- self._deliver(retry, store, evaluation_key="eval-1")
+
+ with recorded_metrics() as metrics:
+ self._deliver(FakeTransport(), store, evaluation_key="eval-1")
+ self._deliver(retry, store, evaluation_key="eval-1")
assert retry.sends == []
+ assert [(m.attributes, m.value) for m in metrics if m.name == "alerts_platform_deliveries_total"] == [
+ ({"source": "logs", "provider": "fake", "path": "native", "outcome": "sent"}, 1),
+ ({"source": "logs", "provider": "fake", "path": "native", "outcome": "already_delivered"}, 1),
+ ]
+ (lag,) = (m for m in metrics if m.name == "alerts_platform_delivery_lag_ms")
+ assert lag.attributes == {"source": "logs", "provider": "fake"} and lag.value > 0
@parameterized.expand(
[
@@ -279,10 +288,29 @@ class TestDeliveryDispatch(SimpleTestCase):
assert retry.sends == []
assert [handle for _, handle in later.sends] == [None]
+ def test_a_send_outside_a_worker_is_counted_in_the_process_registry(self) -> None:
+ labels = {"source": "logs", "provider": "fake", "path": "native", "outcome": "sent"}
+ before = REGISTRY.get_sample_value("alerts_platform_deliveries_total", labels) or 0
+ lag_before = (
+ REGISTRY.get_sample_value("alerts_platform_delivery_lag_ms_count", {"source": "logs", "provider": "fake"})
+ or 0
+ )
+
+ self._deliver(FakeTransport(), RecordingThreadStore())
+
+ assert REGISTRY.get_sample_value("alerts_platform_deliveries_total", labels) == before + 1
+ assert (
+ REGISTRY.get_sample_value("alerts_platform_delivery_lag_ms_count", {"source": "logs", "provider": "fake"})
+ == lag_before + 1
+ )
+
def test_a_provider_outage_is_counted_as_a_failed_delivery(self) -> None:
transport = FakeTransport(error=ConnectionError("slack unreachable"))
- with patch("products.alerts_platform.backend.delivery.dispatch.record_delivery") as recorded:
+ with (
+ patch("products.alerts_platform.backend.delivery.dispatch.record_delivery") as recorded,
+ recorded_metrics() as metrics,
+ ):
with pytest.raises(ConnectionError):
deliver(
transport=transport,
@@ -298,6 +326,7 @@ class TestDeliveryDispatch(SimpleTestCase):
assert recorded.call_args.kwargs["succeeded"] is False
# Its text can carry a credential URL, so only the class reaches the person's log.
assert recorded.call_args.kwargs["error"] == "ConnectionError"
+ assert [m.attributes["outcome"] for m in metrics if m.name == "alerts_platform_deliveries_total"] == ["failed"]
def test_a_store_that_remembers_nothing_still_delivers(self) -> None:
transport = FakeTransport()
diff --git a/products/alerts_platform/backend/tests/test_delivery_evaluation.py b/products/alerts_platform/backend/tests/test_delivery_evaluation.py
index 99719dc506d5..0fd3438d66b7 100644
--- a/products/alerts_platform/backend/tests/test_delivery_evaluation.py
+++ b/products/alerts_platform/backend/tests/test_delivery_evaluation.py
@@ -34,6 +34,7 @@ from products.alerts_platform.backend.facade.contracts import (
SourceKind,
)
from products.alerts_platform.backend.models import PlatformAlertConfiguration
+from products.alerts_platform.backend.tests.recorded_metrics import recorded_metrics
_MODULE = "products.alerts_platform.backend.delivery.evaluation"
_LEGACY_MODULE = "products.alerts_platform.backend.delivery.legacy_webhook"
@@ -685,17 +686,23 @@ class TestDeliveryPath(APIBaseTest):
)
def test_a_rerun_after_a_refused_subscriber_send_emits_once(self) -> None:
- for _ in range(2):
- with pytest.raises(DeliveryError):
- self._run(
- _announcement(_transition(AlertEventKind.FIRING)),
- self._chooses(DeliveryPath.HOG_FUNCTION),
- groups=[HOG_FUNCTION_SLACK, SUBSCRIBER],
- transport=RefusingTransport,
- real_thread_store=True,
- )
+ with recorded_metrics() as metrics:
+ for _ in range(2):
+ with pytest.raises(DeliveryError):
+ self._run(
+ _announcement(_transition(AlertEventKind.FIRING)),
+ self._chooses(DeliveryPath.HOG_FUNCTION),
+ groups=[HOG_FUNCTION_SLACK, SUBSCRIBER],
+ transport=RefusingTransport,
+ real_thread_store=True,
+ )
assert len(self.emitted) == 1
+ assert [
+ (m.attributes["outcome"], m.value)
+ for m in metrics
+ if m.name == "alerts_platform_deliveries_total" and m.attributes["path"] == "hog_function"
+ ] == [("sent", 1), ("already_delivered", 1)]
def test_a_held_subscriber_thread_holds_the_emission_for_the_rerun(self) -> None:
with pytest.raises(ThreadBusy):
diff --git a/products/alerts_platform/backend/tests/test_inventory.py b/products/alerts_platform/backend/tests/test_inventory.py
index d786ead2b275..d7d7e85391b5 100644
--- a/products/alerts_platform/backend/tests/test_inventory.py
+++ b/products/alerts_platform/backend/tests/test_inventory.py
@@ -10,8 +10,13 @@ from django.db import OperationalError, connection
from posthog.models import Team
from products.alerts_platform.backend.facade.contracts import SourceKind
-from products.alerts_platform.backend.facade.enums import PlatformAlertState
-from products.alerts_platform.backend.logic.inventory import AlertCount, SourceInventory, count_inventory
+from products.alerts_platform.backend.facade.enums import PlatformAlertOwner, PlatformAlertState
+from products.alerts_platform.backend.logic.inventory import (
+ AlertCount,
+ ConfigurationCount,
+ SourceInventory,
+ count_inventory,
+)
from products.alerts_platform.backend.models import PlatformAlert, PlatformAlertConfiguration
from products.alerts_platform.backend.temporal.inventory_metrics import push_inventory
@@ -27,6 +32,7 @@ class TestSourceInventory(BaseTest):
*,
source: SourceKind = SourceKind.LOGS,
enabled: bool = True,
+ owner: PlatformAlertOwner = PlatformAlertOwner.SOURCE,
snooze_until: datetime | None = None,
alerts: list[tuple[str, datetime | None]] | None = None,
) -> None:
@@ -34,6 +40,7 @@ class TestSourceInventory(BaseTest):
team=team,
name="alert",
enabled=enabled,
+ owner=owner,
source_kind=source.value,
source_config={},
check_interval_minutes=5,
@@ -55,7 +62,7 @@ class TestSourceInventory(BaseTest):
alerts=[("firing", None), ("firing", LATER), ("not_firing", EARLIER)],
)
self._configuration(other_team, snooze_until=LATER, alerts=[("firing", None)])
- self._configuration(other_team, alerts=[("broken", None)])
+ self._configuration(other_team, owner=PlatformAlertOwner.PLATFORM, alerts=[("broken", None)])
self._configuration(other_team, enabled=False, alerts=[("firing", None)])
self._configuration(self.team, source=SourceKind.INSIGHT, alerts=[("firing", None)])
@@ -63,7 +70,12 @@ class TestSourceInventory(BaseTest):
assert set(by_source) == set(SourceKind)
logs = by_source[SourceKind.LOGS]
- assert (logs.enabled_configurations, logs.disabled_configurations) == (3, 1)
+ assert {(c.enabled, c.owner.value): c.count for c in logs.configurations} == {
+ (True, "source"): 2,
+ (True, "platform"): 1,
+ (False, "source"): 1,
+ (False, "platform"): 0,
+ }
assert {(a.state.value, a.muted): a.count for a in logs.alerts if a.count} == {
("firing", False): 1,
("firing", True): 2,
@@ -72,7 +84,7 @@ class TestSourceInventory(BaseTest):
}
assert len(logs.alerts) == len(PlatformAlertState) * 2
insight = by_source[SourceKind.INSIGHT]
- assert (insight.enabled_configurations, insight.disabled_configurations) == (1, 0)
+ assert {(c.enabled, c.owner.value): c.count for c in insight.configurations if c.count} == {(True, "source"): 1}
assert {(a.state.value, a.muted): a.count for a in insight.alerts if a.count} == {("firing", False): 1}
def test_a_slow_count_is_canceled_by_the_statement_timeout(self) -> None:
@@ -100,8 +112,11 @@ class TestSourceInventory(BaseTest):
def test_push_replaces_the_group_with_the_dashboard_labels() -> None:
inventory = SourceInventory(
source=SourceKind.LOGS,
- enabled_configurations=3,
- disabled_configurations=1,
+ configurations=(
+ ConfigurationCount(enabled=True, owner=PlatformAlertOwner.SOURCE, count=3),
+ ConfigurationCount(enabled=True, owner=PlatformAlertOwner.PLATFORM, count=2),
+ ConfigurationCount(enabled=False, owner=PlatformAlertOwner.SOURCE, count=1),
+ ),
alerts=(AlertCount(state=PlatformAlertState.FIRING, muted=True, count=2),),
)
@@ -111,8 +126,16 @@ def test_push_replaces_the_group_with_the_dashboard_labels() -> None:
assert push.call_args.args[0] == "alerts_platform_inventory"
assert push.call_args.kwargs == {"replace_group": True, "timeout": 5}
registry = push.call_args.args[1]
- assert registry.get_sample_value("alerts_platform_configurations", {"source": "logs", "enabled": "true"}) == 3
- assert registry.get_sample_value("alerts_platform_configurations", {"source": "logs", "enabled": "false"}) == 1
+ configurations = {
+ (labels["enabled"], labels["owner"]): value
+ for labels, value in (
+ (sample.labels, sample.value)
+ for metric in registry.collect()
+ if metric.name == "alerts_platform_configurations"
+ for sample in metric.samples
+ )
+ }
+ assert configurations == {("true", "source"): 3, ("true", "platform"): 2, ("false", "source"): 1}
assert (
registry.get_sample_value("alerts_platform_alerts", {"source": "logs", "state": "firing", "muted": "true"}) == 2
)