Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ci): alert support hero when LLM cost PRs lag (#125054)

Charles Vien25e4741fc4db
diff --git a/.github/owners.yaml b/.github/owners.yaml
index cb69fdd643d4..dbe8fef846f7 100644
--- a/.github/owners.yaml
+++ b/.github/owners.yaml
@@ -27,6 +27,7 @@ rules:
- '/workflows/ci-llm-gateway.yml'
- '/workflows/llm-gateway-cd.yml'
- '/workflows/update-ai-costs.yml'
+ - '/workflows/update-ai-costs-stale-alert.yml'
owners: [team-devex, team-agent-infrastructure]
- match: ['/workflows/ci-llm-gateway.yml', '/workflows/llm-gateway-cd.yml']
sensitive: true
diff --git a/.github/scripts/update-ai-costs-stale-alert.js b/.github/scripts/update-ai-costs-stale-alert.js
new file mode 100644
index 000000000000..fc873a32da8f
--- /dev/null
+++ b/.github/scripts/update-ai-costs-stale-alert.js
@@ -0,0 +1,112 @@
+// Pings #support-agent-infra until the automated LLM cost PR merges. Runs hourly on weekdays.
+//
+// Pings when the PR is posted, again at 8h, then every 24h.
+// The clock starts at the first unmerged PR, since each cost refresh replaces the open one.
+// The support hero is always tagged. AI observability joins once new prices have waited 24h.
+// Each run covers the time since the last successful run, so a missed run catches up on the next.
+
+const COST_PR_BRANCH = 'chore/llma-update-ai-costs'
+const HOUR_MS = 60 * 60 * 1000
+const DAY_MS = 24 * HOUR_MS
+const REPLACEMENT_WINDOW_MS = 30 * 60 * 1000
+const EARLY_ALERT_HOURS = [0, 8]
+const AI_OBSERVABILITY_AFTER_HOURS = 24
+
+function findWaitingSince(prs) {
+ const [current, ...older] = prs
+ let waitingSince = Date.parse(current.created_at)
+ let prCount = 1
+ for (const pr of older) {
+ const replacedImmediately = Math.abs(waitingSince - Date.parse(pr.closed_at)) < REPLACEMENT_WINDOW_MS
+ if (pr.merged_at || !replacedImmediately) {
+ break
+ }
+ waitingSince = Date.parse(pr.created_at)
+ prCount++
+ }
+ return { waitingSince, prCount }
+}
+
+function isAlertDue({ waitingSince, lastCheckAt, thisCheckAt }) {
+ const nextDailyAlert = waitingSince + Math.max(1, Math.floor((lastCheckAt - waitingSince) / DAY_MS) + 1) * DAY_MS
+ const alertTimes = [...EARLY_ALERT_HOURS.map((hours) => waitingSince + hours * HOUR_MS), nextDailyAlert]
+ return alertTimes.some((time) => time > lastCheckAt && time <= thisCheckAt)
+}
+
+function buildAlertText({ pr, waitingSince, prCount, now, supportHeroGroup, aiObservabilityGroup }) {
+ const waitedHours = Math.floor((now - waitingSince) / HOUR_MS)
+ const groups =
+ waitedHours >= AI_OBSERVABILITY_AFTER_HOURS ? [supportHeroGroup, aiObservabilityGroup] : [supportHeroGroup]
+ const mentions = groups.map((group) => `<!subteam^${group}>`).join(' ')
+ const link = `<${pr.html_url}|#${pr.number}>`
+ if (waitedHours === 0) {
+ return `${mentions} New LLM cost update ${link} needs to merge.`
+ }
+ const history = prCount > 1 ? ` across ${prCount} PRs` : ''
+ return `${mentions} LLM cost update ${link} has not merged. New prices have waited ${waitedHours}h${history}.`
+}
+
+async function findCheckWindow({ github, owner, repo, runId }) {
+ const { data: thisRun } = await github.rest.actions.getWorkflowRun({ owner, repo, run_id: runId })
+ const thisCheckAt = Date.parse(thisRun.created_at)
+ const { data: successfulRuns } = await github.rest.actions.listWorkflowRuns({
+ owner,
+ repo,
+ workflow_id: thisRun.workflow_id,
+ status: 'success',
+ per_page: 10,
+ })
+ const lastSuccess = successfulRuns.workflow_runs.find(
+ (run) => run.id !== thisRun.id && Date.parse(run.created_at) < thisCheckAt
+ )
+ const lastCheckAt = lastSuccess ? Date.parse(lastSuccess.created_at) : thisCheckAt - HOUR_MS
+ return { lastCheckAt, thisCheckAt }
+}
+
+async function listCostPrsSinceLastMerge({ github, owner, repo }) {
+ return github.paginate(
+ github.rest.pulls.list,
+ {
+ owner,
+ repo,
+ head: `${owner}:${COST_PR_BRANCH}`,
+ state: 'all',
+ sort: 'created',
+ direction: 'desc',
+ per_page: 100,
+ },
+ (response, done) => {
+ if (response.data.some((pr) => pr.merged_at)) {
+ done()
+ }
+ return response.data
+ }
+ )
+}
+
+async function checkStaleCostPr({ github, context, core }, { supportHeroGroup, aiObservabilityGroup }) {
+ const { owner, repo } = context.repo
+ const { lastCheckAt, thisCheckAt } = await findCheckWindow({ github, owner, repo, runId: context.runId })
+
+ const prs = await listCostPrsSinceLastMerge({ github, owner, repo })
+ const pr = prs[0]
+ if (!pr || pr.state !== 'open') {
+ core.info('No open LLM cost PR')
+ return
+ }
+
+ const { waitingSince, prCount } = findWaitingSince(prs)
+ if (!isAlertDue({ waitingSince, lastCheckAt, thisCheckAt })) {
+ core.info(`#${pr.number} is open: no alert due`)
+ return
+ }
+
+ const text = buildAlertText({ pr, waitingSince, prCount, now: Date.now(), supportHeroGroup, aiObservabilityGroup })
+ core.info(text)
+ core.setOutput('text', text)
+}
+
+module.exports = checkStaleCostPr
+module.exports.findWaitingSince = findWaitingSince
+module.exports.isAlertDue = isAlertDue
+module.exports.buildAlertText = buildAlertText
diff --git a/.github/scripts/update-ai-costs-stale-alert.test.js b/.github/scripts/update-ai-costs-stale-alert.test.js
new file mode 100644
index 000000000000..63e7fb25c984
--- /dev/null
+++ b/.github/scripts/update-ai-costs-stale-alert.test.js
@@ -0,0 +1,118 @@
+const { describe, it } = require('node:test')
+const assert = require('node:assert/strict')
+
+const { findWaitingSince, isAlertDue, buildAlertText } = require('./update-ai-costs-stale-alert')
+
+const HOUR_MS = 60 * 60 * 1000
+const OPENED_AT = Date.parse('2026-10-20T10:05:00Z')
+const at = (hours) => OPENED_AT + hours * HOUR_MS
+const iso = (hours) => new Date(at(hours)).toISOString()
+
+const costPr = ({ number, openedHours, closedHours = null, merged = false }) => ({
+ number,
+ html_url: `https://example.com/pull/${number}`,
+ created_at: iso(openedHours),
+ closed_at: closedHours === null ? null : iso(closedHours),
+ merged_at: merged ? iso(closedHours) : null,
+})
+
+describe('update-ai-costs-stale-alert', () => {
+ describe('isAlertDue', () => {
+ const cases = [
+ { name: 'sends an alert when the PR opens', from: -0.5, until: 0.5, due: true },
+ { name: 'stays quiet before the PR opens', from: -1.5, until: -0.5, due: false },
+ { name: 'does not repeat an alert on the window edge', from: 0, until: 1, due: false },
+ { name: 'stays quiet between the posted and 8h alerts', from: 3.5, until: 4.5, due: false },
+ { name: 'sends the 8h alert after a dropped run', from: 6.5, until: 8.5, due: true },
+ { name: 'sends the first daily alert at 24h', from: 23.5, until: 24.5, due: true },
+ { name: 'stays quiet between daily alerts', from: 30.5, until: 31.5, due: false },
+ { name: 'sends a later daily alert at 72h', from: 71.5, until: 72.5, due: true },
+ { name: 'catches up after a weekend without runs', from: 30, until: 92, due: true },
+ ]
+ for (const { name, from, until, due } of cases) {
+ it(name, () => {
+ assert.equal(
+ isAlertDue({ waitingSince: OPENED_AT, lastCheckAt: at(from), thisCheckAt: at(until) }),
+ due
+ )
+ })
+ }
+ })
+
+ describe('findWaitingSince', () => {
+ const cases = [
+ {
+ name: 'starts at the only open PR',
+ prs: [costPr({ number: 1, openedHours: 0 })],
+ expected: { waitingSince: at(0), prCount: 1 },
+ },
+ {
+ name: 'follows PRs the refresh replaced',
+ prs: [
+ costPr({ number: 3, openedHours: 20 }),
+ costPr({ number: 2, openedHours: 10, closedHours: 19.99 }),
+ costPr({ number: 1, openedHours: 0, closedHours: 9.99 }),
+ ],
+ expected: { waitingSince: at(0), prCount: 3 },
+ },
+ {
+ name: 'stops at a merged PR',
+ prs: [
+ costPr({ number: 3, openedHours: 20 }),
+ costPr({ number: 2, openedHours: 10, closedHours: 19.99 }),
+ costPr({ number: 1, openedHours: 0, closedHours: 9.99, merged: true }),
+ ],
+ expected: { waitingSince: at(10), prCount: 2 },
+ },
+ {
+ name: 'stops at a PR closed long before the next one opened',
+ prs: [costPr({ number: 2, openedHours: 10 }), costPr({ number: 1, openedHours: 0, closedHours: 3 })],
+ expected: { waitingSince: at(10), prCount: 1 },
+ },
+ ]
+ for (const { name, prs, expected } of cases) {
+ it(name, () => {
+ assert.deepEqual(findWaitingSince(prs), expected)
+ })
+ }
+ })
+
+ describe('buildAlertText', () => {
+ const pr = costPr({ number: 7, openedHours: 0 })
+ const cases = [
+ {
+ name: 'announces a newly posted PR',
+ nowHours: 0.25,
+ prCount: 1,
+ expected: '<!subteam^HERO> New LLM cost update <https://example.com/pull/7|#7> needs to merge.',
+ },
+ {
+ name: 'tags only the support hero before 24h',
+ nowHours: 23,
+ prCount: 3,
+ expected:
+ '<!subteam^HERO> LLM cost update <https://example.com/pull/7|#7> has not merged. New prices have waited 23h across 3 PRs.',
+ },
+ {
+ name: 'adds AI observability from 24h',
+ nowHours: 24,
+ prCount: 1,
+ expected:
+ '<!subteam^HERO> <!subteam^AIO> LLM cost update <https://example.com/pull/7|#7> has not merged. New prices have waited 24h.',
+ },
+ ]
+ for (const { name, nowHours, prCount, expected } of cases) {
+ it(name, () => {
+ const text = buildAlertText({
+ pr,
+ waitingSince: OPENED_AT,
+ prCount,
+ now: at(nowHours),
+ supportHeroGroup: 'HERO',
+ aiObservabilityGroup: 'AIO',
+ })
+ assert.equal(text, expected)
+ })
+ }
+ })
+})
diff --git a/.github/workflows/update-ai-costs-stale-alert.yml b/.github/workflows/update-ai-costs-stale-alert.yml
new file mode 100644
index 000000000000..3e97b52a4364
--- /dev/null
+++ b/.github/workflows/update-ai-costs-stale-alert.yml
@@ -0,0 +1,53 @@
+name: Update AI Costs Stale Alert
+
+on:
+ workflow_dispatch:
+ schedule:
+ - cron: '20 * * * 1-5'
+
+permissions:
+ actions: read
+ contents: read
+ pull-requests: read
+
+concurrency:
+ group: ${{ github.workflow }}
+ cancel-in-progress: false
+
+env:
+ SLACK_CHANNEL: 'C0CB80UM50Q'
+
+jobs:
+ alert-stale-pr:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 5
+
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ persist-credentials: false
+ sparse-checkout: .github/scripts/update-ai-costs-stale-alert.js
+ sparse-checkout-cone-mode: false
+
+ - name: Check the open LLM cost PR
+ id: check
+ uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+ with:
+ script: |
+ const checkStaleCostPr = require('./.github/scripts/update-ai-costs-stale-alert.js')
+ await checkStaleCostPr(
+ { github, context, core },
+ { supportHeroGroup: 'S0CDU0BNFJ4', aiObservabilityGroup: 'S097J2WH3QX' }
+ )
+
+ - name: Post to Slack
+ if: steps.check.outputs.text != ''
+ uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1
+ with:
+ method: chat.postMessage
+ token: ${{ secrets.SLACK_DEVEX_BOT_TOKEN }}
+ errors: true
+ payload: |
+ channel: "${{ env.SLACK_CHANNEL }}"
+ unfurl_links: false
+ text: "${{ steps.check.outputs.text }}"