Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(slo): daily burn rate regression alert to #alerts-product-analytics (#119357)

Vasco de Krijger0ffe32008c87
diff --git a/terraform/providers_root.tf.tpl b/terraform/providers_root.tf.tpl
index e534da4016cf..365ccb1544e9 100644
--- a/terraform/providers_root.tf.tpl
+++ b/terraform/providers_root.tf.tpl
@@ -3,7 +3,7 @@ terraform {
required_providers {
posthog = {
source = "PostHog/posthog"
- version = "1.0.7"
+ version = "1.0.24"
}
}
}
diff --git a/terraform/us/project-2/team-analytics-platform/slo-monitoring/alerts.tf b/terraform/us/project-2/team-analytics-platform/slo-monitoring/alerts.tf
index 6484cbdd4d2e..b25e53e7bbd6 100644
--- a/terraform/us/project-2/team-analytics-platform/slo-monitoring/alerts.tf
+++ b/terraform/us/project-2/team-analytics-platform/slo-monitoring/alerts.tf
@@ -12,3 +12,87 @@ resource "posthog_alert" "alert_delivery_slo" {
insight = each.value.id
subscribed_users = var.analytics_platform_alert_subscribed_user_ids
}
+
+resource "posthog_alert" "slo_regression" {
+ name = "SLO regression: daily burn rate > 3x baseline"
+ enabled = true
+ calculation_interval = "daily"
+ condition_type = "absolute_value"
+ threshold_type = "absolute"
+ threshold_upper = 3
+ evaluation = "any_row"
+ column = "regression_ratio"
+ label_column = "operation"
+ insight = posthog_insight.slo_regression.id
+ subscribed_users = var.analytics_platform_alert_subscribed_user_ids
+}
+
+resource "posthog_hog_function" "slo_regression_slack" {
+ name = "Post SLO regression alert to #alerts-product-analytics"
+ description = "Post to a Slack channel when the SLO regression alert fires"
+ type = "internal_destination"
+ enabled = true
+ hog = "let res := fetch('https://slack.com/api/chat.postMessage', {\n 'body': {\n 'channel': inputs.channel,\n 'icon_emoji': inputs.icon_emoji,\n 'username': inputs.username,\n 'blocks': inputs.blocks,\n 'text': inputs.text\n },\n 'method': 'POST',\n 'headers': {\n 'Authorization': f'Bearer {inputs.slack_workspace.access_token}',\n 'Content-Type': 'application/json'\n }\n});\n\nif (res.status != 200 or res.body.ok == false) {\n throw Error(f'Failed to post message to Slack: {res.status}: {res.body}');\n}"
+
+ inputs_json = jsonencode({
+ "text" = {
+ "value" = "Alert triggered: {event.properties.insight_name}"
+ "templating" = "hog"
+ }
+ "blocks" = {
+ "value" = [
+ {
+ "text" = {
+ "text" = "Alert '{event.properties.alert_name}' firing for insight '{event.properties.insight_name}'"
+ "type" = "plain_text"
+ }
+ "type" = "header"
+ },
+ {
+ "text" = { "text" = "{event.properties.breaches}", "type" = "plain_text" }
+ "type" = "section"
+ },
+ {
+ "type" = "context"
+ "elements" = [{ "text" = "Project: <{project.url}|{project.name}>", "type" = "mrkdwn" }]
+ },
+ { "type" = "divider" },
+ {
+ "type" = "actions"
+ "elements" = [
+ {
+ "url" = "{project.url}/insights/{event.properties.insight_id}"
+ "text" = { "text" = "View Insight", "type" = "plain_text" }
+ "type" = "button"
+ },
+ {
+ "url" = "{project.url}/insights/{event.properties.insight_id}/alerts?alert_id={event.properties.alert_id}"
+ "text" = { "text" = "View Alert", "type" = "plain_text" }
+ "type" = "button"
+ }
+ ]
+ }
+ ]
+ "templating" = "hog"
+ }
+ # #alerts-product-analytics
+ "channel" = { "value" = "C05G937QY1Z", "templating" = "hog" }
+ "username" = { "value" = "SLO-monitor", "templating" = "hog" }
+ "icon_emoji" = { "value" = ":hogzilla:", "templating" = "hog" }
+ "slack_workspace" = { "value" = var.analytics_platform_slack_workspace_id, "templating" = "hog" }
+ })
+
+ filters_json = jsonencode({
+ "source" = "events"
+ "events" = [{ "id" = "$insight_alert_firing", "type" = "events" }]
+ "properties" = [{
+ "key" = "alert_id"
+ "type" = "event"
+ "value" = posthog_alert.slo_regression.id
+ "operator" = "exact"
+ }]
+ })
+
+ template_id = "template-slack"
+ icon_url = "/static/services/slack.png"
+}
diff --git a/terraform/us/project-2/team-analytics-platform/slo-monitoring/insights.tf b/terraform/us/project-2/team-analytics-platform/slo-monitoring/insights.tf
index bf094e9dbc29..0d4fe581df24 100644
--- a/terraform/us/project-2/team-analytics-platform/slo-monitoring/insights.tf
+++ b/terraform/us/project-2/team-analytics-platform/slo-monitoring/insights.tf
@@ -667,3 +667,84 @@ resource "posthog_insight" "alert_delivery_failure_rate" {
tags = ["managed-by:terraform", "slo"]
}
+
+# ---------------------------------------------------------------------------
+# Daily burn rate vs each operation's own recent baseline. Drives the
+# slo_regression alert, so it covers every operation/region emitting SLO events,
+# not just the ones in local.slo_operations.
+# ---------------------------------------------------------------------------
+resource "posthog_insight" "slo_regression" {
+ name = "SLO: Daily burn rate regression by operation"
+ description = "Yesterday's burn rate (99.95% target) / median of the prior 14 days, floored at 1x. 0 when yesterday had <100 operations or <10 failures."
+ query_json = jsonencode({
+ kind = "DataVisualizationNode"
+ source = {
+ kind = "HogQLQuery"
+ query = <<-SQL
+ -- Same pairing and sample-rate weighting as the burn rate tiles. Yesterday's correlated
+ -- starts are paired with completions from today, so operations in flight at midnight
+ -- aren't counted as failures.
+ WITH per_cid AS (
+ SELECT
+ properties.region AS region,
+ properties.operation AS operation,
+ coalesce(nullIf(properties.correlation_id, ''), '') AS cid,
+ toDate(timestamp) AS d,
+ sumIf(1.0 / coalesce(toFloat(properties.sample_rate), 1.0), event = 'slo_operation_started') AS starts,
+ sumIf(1.0 / coalesce(toFloat(properties.sample_rate), 1.0), event = 'slo_operation_completed' AND properties.outcome = 'success') AS successes,
+ max(1.0 / coalesce(toFloat(properties.sample_rate), 1.0)) AS w,
+ min(if(event = 'slo_operation_started', timestamp, NULL)) AS first_start
+ FROM events
+ WHERE event IN ('slo_operation_started', 'slo_operation_completed')
+ AND properties.region IN ('US', 'EU')
+ AND timestamp >= today() - 17
+ GROUP BY region, operation, cid, d
+ ),
+ daily AS (
+ SELECT region, operation, d, sum(total) AS total, sum(failures) AS failures
+ FROM (
+ SELECT region, operation, d, starts AS total, greatest(starts - successes, 0.0) AS failures
+ FROM per_cid
+ WHERE cid = ''
+
+ UNION ALL
+
+ SELECT region, operation, toDate(min(first_start)) AS d, max(w) AS total, if(max(successes) > 0, 0.0, max(w)) AS failures
+ FROM per_cid
+ WHERE cid != ''
+ GROUP BY region, operation, cid
+ HAVING d IS NOT NULL
+ )
+ GROUP BY region, operation, d
+ ),
+ scored AS (
+ SELECT
+ region,
+ operation,
+ d,
+ total,
+ failures,
+ failures / total / 0.0005 AS burn,
+ median(failures / total / 0.0005) OVER (PARTITION BY region, operation ORDER BY d ASC ROWS BETWEEN 14 PRECEDING AND 1 PRECEDING) AS baseline
+ FROM daily
+ )
+ SELECT
+ concat(region, ' / ', operation) AS operation,
+ round(total) AS total,
+ round(failures) AS failures,
+ round(burn, 1) AS burn_rate,
+ round(baseline, 1) AS baseline_burn_rate,
+ -- Floor the baseline at 1x so near-zero baselines don't turn a handful of failures into a huge ratio.
+ round(if(total >= 100 AND failures >= 10, burn / greatest(baseline, 1.0), 0), 2) AS regression_ratio
+ FROM scored
+ WHERE d = yesterday()
+ ORDER BY regression_ratio DESC
+ LIMIT 100
+ SQL
+ }
+ display = "ActionsTable"
+ })
+
+ dashboard_ids = [posthog_dashboard.slo_monitoring.id]
+ tags = ["managed-by:terraform", "slo"]
+}
diff --git a/terraform/us/project-2/team-analytics-platform/slo-monitoring/layouts.tf b/terraform/us/project-2/team-analytics-platform/slo-monitoring/layouts.tf
index aab6c73338a9..971c2c5116dc 100644
--- a/terraform/us/project-2/team-analytics-platform/slo-monitoring/layouts.tf
+++ b/terraform/us/project-2/team-analytics-platform/slo-monitoring/layouts.tf
@@ -21,6 +21,15 @@ resource "posthog_dashboard_layout" "slo_monitoring" {
}
})
},
+ {
+ insight_id = posthog_insight.slo_regression.id
+ layouts_json = jsonencode({
+ sm = {
+ h = 4, i = "regression", w = 12, x = 0, y = 6
+ minH = 2, minW = 2, moved = false, static = false
+ }
+ })
+ },
{
text_body = "**Success rate** is a 28-day rolling window showing the percentage of successful operations. **Burn rate** shows how fast you're consuming your error budget on a logarithmic scale — the Budget rate line marks sustainable consumption, crossing 100x burn indicates an active incident. **Duration** shows p50/p95/p99 operation latency in seconds."
layouts_json = jsonencode({
@@ -33,7 +42,7 @@ resource "posthog_dashboard_layout" "slo_monitoring" {
],
# Per-operation sections: header + burn rate (left) + duration (right) per region.
# Each section: header (h=1) + N regions * row_height (h=3).
- # Base y = 6 (after success rate h=5 + text h=1).
+ # Base y = 10 (after success rate h=5 + text h=1 + regression table h=4).
# NOTE: y offset formula assumes all operations have the same number of regions.
flatten([
for op_idx, op_key in sort(keys(local.slo_operations)) : concat(
@@ -43,7 +52,7 @@ resource "posthog_dashboard_layout" "slo_monitoring" {
layouts_json = jsonencode({
sm = {
h = 1, i = "header_${op_key}", w = 12, x = 0,
- y = 6 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3)
+ y = 10 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3)
minH = 1, minW = 1, moved = false, static = false
}
})
@@ -55,7 +64,7 @@ resource "posthog_dashboard_layout" "slo_monitoring" {
layouts_json = jsonencode({
sm = {
h = 3, i = "burn_${op_key}_${lower(region)}", w = 6, x = 0,
- y = 6 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3) + 1 + reg_idx * 3
+ y = 10 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3) + 1 + reg_idx * 3
minH = 2, minW = 2, moved = false, static = false
}
})
@@ -68,7 +77,7 @@ resource "posthog_dashboard_layout" "slo_monitoring" {
layouts_json = jsonencode({
sm = {
h = 3, i = "dur_${op_key}_${lower(region)}", w = 6, x = 6,
- y = 6 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3) + 1 + reg_idx * 3
+ y = 10 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3) + 1 + reg_idx * 3
minH = 2, minW = 2, moved = false, static = false
}
})