:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ci): alert support hero when LLM cost PRs lag (#125054)
Charles Vien25e4741fc4db
diff --git a/.github/owners.yaml b/.github/owners.yamlindex cb69fdd643d4..dbe8fef846f7 100644--- a/.github/owners.yaml+++ b/.github/owners.yaml@@ -27,6 +27,7 @@ rules:- '/workflows/ci-llm-gateway.yml'- '/workflows/llm-gateway-cd.yml'- '/workflows/update-ai-costs.yml'+ - '/workflows/update-ai-costs-stale-alert.yml'owners: [team-devex, team-agent-infrastructure]- match: ['/workflows/ci-llm-gateway.yml', '/workflows/llm-gateway-cd.yml']sensitive: truediff --git a/.github/scripts/update-ai-costs-stale-alert.js b/.github/scripts/update-ai-costs-stale-alert.jsnew file mode 100644index 000000000000..fc873a32da8f--- /dev/null+++ b/.github/scripts/update-ai-costs-stale-alert.js@@ -0,0 +1,112 @@+// Pings #support-agent-infra until the automated LLM cost PR merges. Runs hourly on weekdays.+//+// Pings when the PR is posted, again at 8h, then every 24h.+// The clock starts at the first unmerged PR, since each cost refresh replaces the open one.+// The support hero is always tagged. AI observability joins once new prices have waited 24h.+// Each run covers the time since the last successful run, so a missed run catches up on the next.++const COST_PR_BRANCH = 'chore/llma-update-ai-costs'+const HOUR_MS = 60 * 60 * 1000+const DAY_MS = 24 * HOUR_MS+const REPLACEMENT_WINDOW_MS = 30 * 60 * 1000+const EARLY_ALERT_HOURS = [0, 8]+const AI_OBSERVABILITY_AFTER_HOURS = 24++function findWaitingSince(prs) {+ const [current, ...older] = prs+ let waitingSince = Date.parse(current.created_at)+ let prCount = 1+ for (const pr of older) {+ const replacedImmediately = Math.abs(waitingSince - Date.parse(pr.closed_at)) < REPLACEMENT_WINDOW_MS+ if (pr.merged_at || !replacedImmediately) {+ break+ }+ waitingSince = Date.parse(pr.created_at)+ prCount+++ }+ return { waitingSince, prCount }+}++function isAlertDue({ waitingSince, lastCheckAt, thisCheckAt }) {+ const nextDailyAlert = waitingSince + Math.max(1, Math.floor((lastCheckAt - waitingSince) / DAY_MS) + 1) * DAY_MS+ const alertTimes = [...EARLY_ALERT_HOURS.map((hours) => waitingSince + hours * HOUR_MS), nextDailyAlert]+ return alertTimes.some((time) => time > lastCheckAt && time <= thisCheckAt)+}++function buildAlertText({ pr, waitingSince, prCount, now, supportHeroGroup, aiObservabilityGroup }) {+ const waitedHours = Math.floor((now - waitingSince) / HOUR_MS)+ const groups =+ waitedHours >= AI_OBSERVABILITY_AFTER_HOURS ? [supportHeroGroup, aiObservabilityGroup] : [supportHeroGroup]+ const mentions = groups.map((group) => `<!subteam^${group}>`).join(' ')+ const link = `<${pr.html_url}|#${pr.number}>`+ if (waitedHours === 0) {+ return `${mentions} New LLM cost update ${link} needs to merge.`+ }+ const history = prCount > 1 ? ` across ${prCount} PRs` : ''+ return `${mentions} LLM cost update ${link} has not merged. New prices have waited ${waitedHours}h${history}.`+}++async function findCheckWindow({ github, owner, repo, runId }) {+ const { data: thisRun } = await github.rest.actions.getWorkflowRun({ owner, repo, run_id: runId })+ const thisCheckAt = Date.parse(thisRun.created_at)+ const { data: successfulRuns } = await github.rest.actions.listWorkflowRuns({+ owner,+ repo,+ workflow_id: thisRun.workflow_id,+ status: 'success',+ per_page: 10,+ })+ const lastSuccess = successfulRuns.workflow_runs.find(+ (run) => run.id !== thisRun.id && Date.parse(run.created_at) < thisCheckAt+ )+ const lastCheckAt = lastSuccess ? Date.parse(lastSuccess.created_at) : thisCheckAt - HOUR_MS+ return { lastCheckAt, thisCheckAt }+}++async function listCostPrsSinceLastMerge({ github, owner, repo }) {+ return github.paginate(+ github.rest.pulls.list,+ {+ owner,+ repo,+ head: `${owner}:${COST_PR_BRANCH}`,+ state: 'all',+ sort: 'created',+ direction: 'desc',+ per_page: 100,+ },+ (response, done) => {+ if (response.data.some((pr) => pr.merged_at)) {+ done()+ }+ return response.data+ }+ )+}++async function checkStaleCostPr({ github, context, core }, { supportHeroGroup, aiObservabilityGroup }) {+ const { owner, repo } = context.repo+ const { lastCheckAt, thisCheckAt } = await findCheckWindow({ github, owner, repo, runId: context.runId })++ const prs = await listCostPrsSinceLastMerge({ github, owner, repo })+ const pr = prs[0]+ if (!pr || pr.state !== 'open') {+ core.info('No open LLM cost PR')+ return+ }++ const { waitingSince, prCount } = findWaitingSince(prs)+ if (!isAlertDue({ waitingSince, lastCheckAt, thisCheckAt })) {+ core.info(`#${pr.number} is open: no alert due`)+ return+ }++ const text = buildAlertText({ pr, waitingSince, prCount, now: Date.now(), supportHeroGroup, aiObservabilityGroup })+ core.info(text)+ core.setOutput('text', text)+}++module.exports = checkStaleCostPr+module.exports.findWaitingSince = findWaitingSince+module.exports.isAlertDue = isAlertDue+module.exports.buildAlertText = buildAlertTextdiff --git a/.github/scripts/update-ai-costs-stale-alert.test.js b/.github/scripts/update-ai-costs-stale-alert.test.jsnew file mode 100644index 000000000000..63e7fb25c984--- /dev/null+++ b/.github/scripts/update-ai-costs-stale-alert.test.js@@ -0,0 +1,118 @@+const { describe, it } = require('node:test')+const assert = require('node:assert/strict')++const { findWaitingSince, isAlertDue, buildAlertText } = require('./update-ai-costs-stale-alert')++const HOUR_MS = 60 * 60 * 1000+const OPENED_AT = Date.parse('2026-10-20T10:05:00Z')+const at = (hours) => OPENED_AT + hours * HOUR_MS+const iso = (hours) => new Date(at(hours)).toISOString()++const costPr = ({ number, openedHours, closedHours = null, merged = false }) => ({+ number,+ html_url: `https://example.com/pull/${number}`,+ created_at: iso(openedHours),+ closed_at: closedHours === null ? null : iso(closedHours),+ merged_at: merged ? iso(closedHours) : null,+})++describe('update-ai-costs-stale-alert', () => {+ describe('isAlertDue', () => {+ const cases = [+ { name: 'sends an alert when the PR opens', from: -0.5, until: 0.5, due: true },+ { name: 'stays quiet before the PR opens', from: -1.5, until: -0.5, due: false },+ { name: 'does not repeat an alert on the window edge', from: 0, until: 1, due: false },+ { name: 'stays quiet between the posted and 8h alerts', from: 3.5, until: 4.5, due: false },+ { name: 'sends the 8h alert after a dropped run', from: 6.5, until: 8.5, due: true },+ { name: 'sends the first daily alert at 24h', from: 23.5, until: 24.5, due: true },+ { name: 'stays quiet between daily alerts', from: 30.5, until: 31.5, due: false },+ { name: 'sends a later daily alert at 72h', from: 71.5, until: 72.5, due: true },+ { name: 'catches up after a weekend without runs', from: 30, until: 92, due: true },+ ]+ for (const { name, from, until, due } of cases) {+ it(name, () => {+ assert.equal(+ isAlertDue({ waitingSince: OPENED_AT, lastCheckAt: at(from), thisCheckAt: at(until) }),+ due+ )+ })+ }+ })++ describe('findWaitingSince', () => {+ const cases = [+ {+ name: 'starts at the only open PR',+ prs: [costPr({ number: 1, openedHours: 0 })],+ expected: { waitingSince: at(0), prCount: 1 },+ },+ {+ name: 'follows PRs the refresh replaced',+ prs: [+ costPr({ number: 3, openedHours: 20 }),+ costPr({ number: 2, openedHours: 10, closedHours: 19.99 }),+ costPr({ number: 1, openedHours: 0, closedHours: 9.99 }),+ ],+ expected: { waitingSince: at(0), prCount: 3 },+ },+ {+ name: 'stops at a merged PR',+ prs: [+ costPr({ number: 3, openedHours: 20 }),+ costPr({ number: 2, openedHours: 10, closedHours: 19.99 }),+ costPr({ number: 1, openedHours: 0, closedHours: 9.99, merged: true }),+ ],+ expected: { waitingSince: at(10), prCount: 2 },+ },+ {+ name: 'stops at a PR closed long before the next one opened',+ prs: [costPr({ number: 2, openedHours: 10 }), costPr({ number: 1, openedHours: 0, closedHours: 3 })],+ expected: { waitingSince: at(10), prCount: 1 },+ },+ ]+ for (const { name, prs, expected } of cases) {+ it(name, () => {+ assert.deepEqual(findWaitingSince(prs), expected)+ })+ }+ })++ describe('buildAlertText', () => {+ const pr = costPr({ number: 7, openedHours: 0 })+ const cases = [+ {+ name: 'announces a newly posted PR',+ nowHours: 0.25,+ prCount: 1,+ expected: '<!subteam^HERO> New LLM cost update <https://example.com/pull/7|#7> needs to merge.',+ },+ {+ name: 'tags only the support hero before 24h',+ nowHours: 23,+ prCount: 3,+ expected:+ '<!subteam^HERO> LLM cost update <https://example.com/pull/7|#7> has not merged. New prices have waited 23h across 3 PRs.',+ },+ {+ name: 'adds AI observability from 24h',+ nowHours: 24,+ prCount: 1,+ expected:+ '<!subteam^HERO> <!subteam^AIO> LLM cost update <https://example.com/pull/7|#7> has not merged. New prices have waited 24h.',+ },+ ]+ for (const { name, nowHours, prCount, expected } of cases) {+ it(name, () => {+ const text = buildAlertText({+ pr,+ waitingSince: OPENED_AT,+ prCount,+ now: at(nowHours),+ supportHeroGroup: 'HERO',+ aiObservabilityGroup: 'AIO',+ })+ assert.equal(text, expected)+ })+ }+ })+})diff --git a/.github/workflows/update-ai-costs-stale-alert.yml b/.github/workflows/update-ai-costs-stale-alert.ymlnew file mode 100644index 000000000000..3e97b52a4364--- /dev/null+++ b/.github/workflows/update-ai-costs-stale-alert.yml@@ -0,0 +1,53 @@+name: Update AI Costs Stale Alert++on:+ workflow_dispatch:+ schedule:+ - cron: '20 * * * 1-5'++permissions:+ actions: read+ contents: read+ pull-requests: read++concurrency:+ group: ${{ github.workflow }}+ cancel-in-progress: false++env:+ SLACK_CHANNEL: 'C0CB80UM50Q'++jobs:+ alert-stale-pr:+ runs-on: ubuntu-24.04+ timeout-minutes: 5++ steps:+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2+ with:+ persist-credentials: false+ sparse-checkout: .github/scripts/update-ai-costs-stale-alert.js+ sparse-checkout-cone-mode: false++ - name: Check the open LLM cost PR+ id: check+ uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0+ with:+ script: |+ const checkStaleCostPr = require('./.github/scripts/update-ai-costs-stale-alert.js')+ await checkStaleCostPr(+ { github, context, core },+ { supportHeroGroup: 'S0CDU0BNFJ4', aiObservabilityGroup: 'S097J2WH3QX' }+ )++ - name: Post to Slack+ if: steps.check.outputs.text != ''+ uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1+ with:+ method: chat.postMessage+ token: ${{ secrets.SLACK_DEVEX_BOT_TOKEN }}+ errors: true+ payload: |+ channel: "${{ env.SLACK_CHANNEL }}"+ unfurl_links: false+ text: "${{ steps.check.outputs.text }}"