:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(slo): daily burn rate regression alert to #alerts-product-analytics (#119357)
Vasco de Krijger0ffe32008c87
diff --git a/terraform/providers_root.tf.tpl b/terraform/providers_root.tf.tplindex e534da4016cf..365ccb1544e9 100644--- a/terraform/providers_root.tf.tpl+++ b/terraform/providers_root.tf.tpl@@ -3,7 +3,7 @@ terraform {required_providers {posthog = {source = "PostHog/posthog"- version = "1.0.7"+ version = "1.0.24"}}}diff --git a/terraform/us/project-2/team-analytics-platform/slo-monitoring/alerts.tf b/terraform/us/project-2/team-analytics-platform/slo-monitoring/alerts.tfindex 6484cbdd4d2e..b25e53e7bbd6 100644--- a/terraform/us/project-2/team-analytics-platform/slo-monitoring/alerts.tf+++ b/terraform/us/project-2/team-analytics-platform/slo-monitoring/alerts.tf@@ -12,3 +12,87 @@ resource "posthog_alert" "alert_delivery_slo" {insight = each.value.idsubscribed_users = var.analytics_platform_alert_subscribed_user_ids}++resource "posthog_alert" "slo_regression" {+ name = "SLO regression: daily burn rate > 3x baseline"+ enabled = true+ calculation_interval = "daily"+ condition_type = "absolute_value"+ threshold_type = "absolute"+ threshold_upper = 3+ evaluation = "any_row"+ column = "regression_ratio"+ label_column = "operation"+ insight = posthog_insight.slo_regression.id+ subscribed_users = var.analytics_platform_alert_subscribed_user_ids+}++resource "posthog_hog_function" "slo_regression_slack" {+ name = "Post SLO regression alert to #alerts-product-analytics"+ description = "Post to a Slack channel when the SLO regression alert fires"+ type = "internal_destination"+ enabled = true+ hog = "let res := fetch('https://slack.com/api/chat.postMessage', {\n 'body': {\n 'channel': inputs.channel,\n 'icon_emoji': inputs.icon_emoji,\n 'username': inputs.username,\n 'blocks': inputs.blocks,\n 'text': inputs.text\n },\n 'method': 'POST',\n 'headers': {\n 'Authorization': f'Bearer {inputs.slack_workspace.access_token}',\n 'Content-Type': 'application/json'\n }\n});\n\nif (res.status != 200 or res.body.ok == false) {\n throw Error(f'Failed to post message to Slack: {res.status}: {res.body}');\n}"++ inputs_json = jsonencode({+ "text" = {+ "value" = "Alert triggered: {event.properties.insight_name}"+ "templating" = "hog"+ }+ "blocks" = {+ "value" = [+ {+ "text" = {+ "text" = "Alert '{event.properties.alert_name}' firing for insight '{event.properties.insight_name}'"+ "type" = "plain_text"+ }+ "type" = "header"+ },+ {+ "text" = { "text" = "{event.properties.breaches}", "type" = "plain_text" }+ "type" = "section"+ },+ {+ "type" = "context"+ "elements" = [{ "text" = "Project: <{project.url}|{project.name}>", "type" = "mrkdwn" }]+ },+ { "type" = "divider" },+ {+ "type" = "actions"+ "elements" = [+ {+ "url" = "{project.url}/insights/{event.properties.insight_id}"+ "text" = { "text" = "View Insight", "type" = "plain_text" }+ "type" = "button"+ },+ {+ "url" = "{project.url}/insights/{event.properties.insight_id}/alerts?alert_id={event.properties.alert_id}"+ "text" = { "text" = "View Alert", "type" = "plain_text" }+ "type" = "button"+ }+ ]+ }+ ]+ "templating" = "hog"+ }+ # #alerts-product-analytics+ "channel" = { "value" = "C05G937QY1Z", "templating" = "hog" }+ "username" = { "value" = "SLO-monitor", "templating" = "hog" }+ "icon_emoji" = { "value" = ":hogzilla:", "templating" = "hog" }+ "slack_workspace" = { "value" = var.analytics_platform_slack_workspace_id, "templating" = "hog" }+ })++ filters_json = jsonencode({+ "source" = "events"+ "events" = [{ "id" = "$insight_alert_firing", "type" = "events" }]+ "properties" = [{+ "key" = "alert_id"+ "type" = "event"+ "value" = posthog_alert.slo_regression.id+ "operator" = "exact"+ }]+ })++ template_id = "template-slack"+ icon_url = "/static/services/slack.png"+}diff --git a/terraform/us/project-2/team-analytics-platform/slo-monitoring/insights.tf b/terraform/us/project-2/team-analytics-platform/slo-monitoring/insights.tfindex bf094e9dbc29..0d4fe581df24 100644--- a/terraform/us/project-2/team-analytics-platform/slo-monitoring/insights.tf+++ b/terraform/us/project-2/team-analytics-platform/slo-monitoring/insights.tf@@ -667,3 +667,84 @@ resource "posthog_insight" "alert_delivery_failure_rate" {tags = ["managed-by:terraform", "slo"]}++# ---------------------------------------------------------------------------+# Daily burn rate vs each operation's own recent baseline. Drives the+# slo_regression alert, so it covers every operation/region emitting SLO events,+# not just the ones in local.slo_operations.+# ---------------------------------------------------------------------------+resource "posthog_insight" "slo_regression" {+ name = "SLO: Daily burn rate regression by operation"+ description = "Yesterday's burn rate (99.95% target) / median of the prior 14 days, floored at 1x. 0 when yesterday had <100 operations or <10 failures."+ query_json = jsonencode({+ kind = "DataVisualizationNode"+ source = {+ kind = "HogQLQuery"+ query = <<-SQL+ -- Same pairing and sample-rate weighting as the burn rate tiles. Yesterday's correlated+ -- starts are paired with completions from today, so operations in flight at midnight+ -- aren't counted as failures.+ WITH per_cid AS (+ SELECT+ properties.region AS region,+ properties.operation AS operation,+ coalesce(nullIf(properties.correlation_id, ''), '') AS cid,+ toDate(timestamp) AS d,+ sumIf(1.0 / coalesce(toFloat(properties.sample_rate), 1.0), event = 'slo_operation_started') AS starts,+ sumIf(1.0 / coalesce(toFloat(properties.sample_rate), 1.0), event = 'slo_operation_completed' AND properties.outcome = 'success') AS successes,+ max(1.0 / coalesce(toFloat(properties.sample_rate), 1.0)) AS w,+ min(if(event = 'slo_operation_started', timestamp, NULL)) AS first_start+ FROM events+ WHERE event IN ('slo_operation_started', 'slo_operation_completed')+ AND properties.region IN ('US', 'EU')+ AND timestamp >= today() - 17+ GROUP BY region, operation, cid, d+ ),+ daily AS (+ SELECT region, operation, d, sum(total) AS total, sum(failures) AS failures+ FROM (+ SELECT region, operation, d, starts AS total, greatest(starts - successes, 0.0) AS failures+ FROM per_cid+ WHERE cid = ''++ UNION ALL++ SELECT region, operation, toDate(min(first_start)) AS d, max(w) AS total, if(max(successes) > 0, 0.0, max(w)) AS failures+ FROM per_cid+ WHERE cid != ''+ GROUP BY region, operation, cid+ HAVING d IS NOT NULL+ )+ GROUP BY region, operation, d+ ),+ scored AS (+ SELECT+ region,+ operation,+ d,+ total,+ failures,+ failures / total / 0.0005 AS burn,+ median(failures / total / 0.0005) OVER (PARTITION BY region, operation ORDER BY d ASC ROWS BETWEEN 14 PRECEDING AND 1 PRECEDING) AS baseline+ FROM daily+ )+ SELECT+ concat(region, ' / ', operation) AS operation,+ round(total) AS total,+ round(failures) AS failures,+ round(burn, 1) AS burn_rate,+ round(baseline, 1) AS baseline_burn_rate,+ -- Floor the baseline at 1x so near-zero baselines don't turn a handful of failures into a huge ratio.+ round(if(total >= 100 AND failures >= 10, burn / greatest(baseline, 1.0), 0), 2) AS regression_ratio+ FROM scored+ WHERE d = yesterday()+ ORDER BY regression_ratio DESC+ LIMIT 100+ SQL+ }+ display = "ActionsTable"+ })++ dashboard_ids = [posthog_dashboard.slo_monitoring.id]+ tags = ["managed-by:terraform", "slo"]+}diff --git a/terraform/us/project-2/team-analytics-platform/slo-monitoring/layouts.tf b/terraform/us/project-2/team-analytics-platform/slo-monitoring/layouts.tfindex aab6c73338a9..971c2c5116dc 100644--- a/terraform/us/project-2/team-analytics-platform/slo-monitoring/layouts.tf+++ b/terraform/us/project-2/team-analytics-platform/slo-monitoring/layouts.tf@@ -21,6 +21,15 @@ resource "posthog_dashboard_layout" "slo_monitoring" {}})},+ {+ insight_id = posthog_insight.slo_regression.id+ layouts_json = jsonencode({+ sm = {+ h = 4, i = "regression", w = 12, x = 0, y = 6+ minH = 2, minW = 2, moved = false, static = false+ }+ })+ },{text_body = "**Success rate** is a 28-day rolling window showing the percentage of successful operations. **Burn rate** shows how fast you're consuming your error budget on a logarithmic scale — the Budget rate line marks sustainable consumption, crossing 100x burn indicates an active incident. **Duration** shows p50/p95/p99 operation latency in seconds."layouts_json = jsonencode({@@ -33,7 +42,7 @@ resource "posthog_dashboard_layout" "slo_monitoring" {],# Per-operation sections: header + burn rate (left) + duration (right) per region.# Each section: header (h=1) + N regions * row_height (h=3).- # Base y = 6 (after success rate h=5 + text h=1).+ # Base y = 10 (after success rate h=5 + text h=1 + regression table h=4).# NOTE: y offset formula assumes all operations have the same number of regions.flatten([for op_idx, op_key in sort(keys(local.slo_operations)) : concat(@@ -43,7 +52,7 @@ resource "posthog_dashboard_layout" "slo_monitoring" {layouts_json = jsonencode({sm = {h = 1, i = "header_${op_key}", w = 12, x = 0,- y = 6 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3)+ y = 10 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3)minH = 1, minW = 1, moved = false, static = false}})@@ -55,7 +64,7 @@ resource "posthog_dashboard_layout" "slo_monitoring" {layouts_json = jsonencode({sm = {h = 3, i = "burn_${op_key}_${lower(region)}", w = 6, x = 0,- y = 6 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3) + 1 + reg_idx * 3+ y = 10 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3) + 1 + reg_idx * 3minH = 2, minW = 2, moved = false, static = false}})@@ -68,7 +77,7 @@ resource "posthog_dashboard_layout" "slo_monitoring" {layouts_json = jsonencode({sm = {h = 3, i = "dur_${op_key}_${lower(region)}", w = 6, x = 6,- y = 6 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3) + 1 + reg_idx * 3+ y = 10 + op_idx * (1 + length(local.slo_operations[op_key].regions) * 3) + 1 + reg_idx * 3minH = 2, minW = 2, moved = false, static = false}})