Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ci): list slow pr workflows in weekly digest (#129522)

Raúl Negrón-Otero4526747d4216
diff --git a/.github/scripts/eng-analytics-weekly-digest.mjs b/.github/scripts/eng-analytics-weekly-digest.mjs
index 8c28c02b43a2..fcbf83321622 100644
--- a/.github/scripts/eng-analytics-weekly-digest.mjs
+++ b/.github/scripts/eng-analytics-weekly-digest.mjs
@@ -1,7 +1,8 @@
// Weekly CI digest for #alerts-devex. It compares the last 7 complete UTC days with the 7 days before.
// Spend covers every repository in the organization. Runner and container build minutes come from
// Depot's usage API. Depot CI comes from the depot_ci_usage endpoint, which counts only repositories
-// with a synced Depot source. Cache storage is not counted.
+// with a synced Depot source. Cache storage is not counted. PR workflow p95 durations come from the
+// workflow_health endpoint, which computes them over successful pull request runs only.
import { pathToFileURL } from 'node:url'
@@ -32,6 +33,12 @@ const BUILD_USD_PER_MINUTE = 0.04
const DAY_MS = 24 * 60 * 60 * 1000
const WEEK_MS = 7 * DAY_MS
+const PR_DURATION_BUDGET_SECONDS = 20 * 60
+// One slow run can move the p95 of a workflow with few runs in the week by minutes.
+const MIN_PERCENTILE_RUNS = 100
+// Slack rejects a message with a section text over 3,000 characters.
+const SLACK_SECTION_MAX_CHARS = 3000
+
function billedMinutes(entry) {
// Protobuf JSON omits a field at its zero default.
const minutes = entry?.minutesBilled ?? 0
@@ -153,6 +160,78 @@ async function readSpend(weekStart, weekEnd) {
return { lastWeek, priorWeek, contract: used == null ? null : { ...contract, used } }
}
+function prWorkflowHealth(dateFrom, dateTo) {
+ return withRetry(() =>
+ api('workflow_health', {
+ date_from: dateFrom.toISOString(),
+ date_to: dateTo.toISOString(),
+ run_scope: 'pull_request',
+ source_id: SOURCE_ID,
+ repo: GITHUB_REPOSITORY,
+ })
+ )
+}
+
+// PR workflows whose p95 is over the budget, slowest first. Null when either week cannot be read.
+async function readSlowWorkflows(weekStart, weekEnd) {
+ return Promise.all([
+ prWorkflowHealth(weekStart, weekEnd),
+ prWorkflowHealth(new Date(weekStart.getTime() - WEEK_MS), weekStart),
+ ])
+ .then(([lastWeek, priorWeek]) => {
+ const priorP95 = new Map(priorWeek.map((w) => [w.workflow_name, w.p95_seconds]))
+ return lastWeek
+ .filter(
+ (w) => w.percentile_run_count >= MIN_PERCENTILE_RUNS && w.p95_seconds > PR_DURATION_BUDGET_SECONDS
+ )
+ .sort((a, b) => b.p95_seconds - a.p95_seconds)
+ .map((w) => ({
+ name: w.workflow_name,
+ url: workflowRunsUrl(w.repo, w.workflow_name),
+ p95: w.p95_seconds,
+ priorP95: priorP95.get(w.workflow_name),
+ }))
+ })
+ .catch(unavailable('PR workflow durations'))
+}
+
+function workflowRunsUrl(repo, workflowName) {
+ const path = [repo.owner, repo.name, 'actions', 'workflows', workflowName].map(encodeURIComponent).join('/')
+ return `${HOST}/project/${PROJECT_ID}/engineering-analytics/repos/${path}`
+}
+
+function slackEscape(text) {
+ return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+}
+
+function splitIntoSections(lines) {
+ const notes = []
+ for (const line of lines) {
+ const last = notes.at(-1)
+ if (last != null && last.length + 1 + line.length <= SLACK_SECTION_MAX_CHARS) {
+ notes[notes.length - 1] = `${last}\n${line}`
+ } else {
+ notes.push(line)
+ }
+ }
+ return notes
+}
+
+function slowWorkflowNotes(slow) {
+ if (slow == null) {
+ return []
+ }
+ const budget = fmtLongDuration(PR_DURATION_BUDGET_SECONDS)
+ if (slow.length === 0) {
+ return [`Every PR workflow had a p95 under ${budget}.`]
+ }
+ const lines = slow.map(({ name, url, p95, priorP95 }) => {
+ const prior = priorP95 ? ` (${fmtLongDuration(priorP95)} the week before)` : ''
+ return `• <${url}|${slackEscape(name)}>: ${fmtLongDuration(p95)}${prior}`
+ })
+ return splitIntoSections([`*PR workflows with a p95 over ${budget}*`, ...lines])
+}
+
function isoDay(date) {
return date.toISOString().slice(0, 10)
}
@@ -379,7 +458,7 @@ export async function main() {
const weekEnd = new Date()
weekEnd.setUTCHours(0, 0, 0, 0)
const weekStart = new Date(weekEnd.getTime() - WEEK_MS)
- const [overview, weeks] = await Promise.all([
+ const [overview, weeks, slowWorkflows] = await Promise.all([
// Every figure carries its value for the 7 days before date_from, so one call covers both weeks.
withRetry(() =>
api('repo_overview', {
@@ -391,12 +470,14 @@ export async function main() {
})
),
readSpend(weekStart, weekEnd),
+ readSlowWorkflows(weekStart, weekEnd),
])
const rows = tableRows(overview, weeks)
if (rows.length === 0) {
throw new Error('No figures to post. Check the GitHub and Depot sources and DEPOT_TOKEN.')
}
- const blocks = buildBlocks(weekStart, weekEnd, rows, spendNotes(weeks, weekEnd))
+ const notes = [...spendNotes(weeks, weekEnd), ...slowWorkflowNotes(slowWorkflows)]
+ const blocks = buildBlocks(weekStart, weekEnd, rows, notes)
if (DRY_RUN) {
console.info(JSON.stringify(blocks, null, 2))
return
diff --git a/.github/scripts/eng-analytics-weekly-digest.test.mjs b/.github/scripts/eng-analytics-weekly-digest.test.mjs
index e0f78a18020f..e2cf10139847 100644
--- a/.github/scripts/eng-analytics-weekly-digest.test.mjs
+++ b/.github/scripts/eng-analytics-weekly-digest.test.mjs
@@ -52,9 +52,28 @@ const DEPOT_CI_USAGE = {
[PRIOR_WEEK]: [{ repo: 'posthog/posthog', estimated_cost_usd: 500 }],
[CONTRACT]: [{ repo: 'posthog/posthog', estimated_cost_usd: 6000 }],
}
+const REPO = { provider: 'github', owner: 'PostHog', name: 'posthog' }
+const WORKFLOW_HEALTH = {
+ [WEEK]: [
+ { repo: REPO, workflow_name: 'Frontend CI', percentile_run_count: 500, p95_seconds: 1000 },
+ { repo: REPO, workflow_name: 'Storybook', percentile_run_count: 500, p95_seconds: 1716 },
+ { repo: REPO, workflow_name: 'E2E <CI>', percentile_run_count: 300, p95_seconds: 2400 },
+ { repo: REPO, workflow_name: 'Rarely run', percentile_run_count: 10, p95_seconds: 3000 },
+ ],
+ [PRIOR_WEEK]: [{ repo: REPO, workflow_name: 'Storybook', percentile_run_count: 500, p95_seconds: 1500 }],
+}
+const WORKFLOWS_URL = 'https://example.com/project/1/engineering-analytics/repos/PostHog/posthog/actions/workflows'
+const SLOW_WORKFLOWS_NOTE = [
+ '*PR workflows with a p95 over 20m*',
+ `• <${WORKFLOWS_URL}/E2E%20%3CCI%3E|E2E &lt;CI&gt;>: 40m`,
+ `• <${WORKFLOWS_URL}/Storybook|Storybook>: 29m (25m the week before)`,
+].join('\n')
let importCount = 0
-async function runDigest(t, { env = {}, depotUsage = DEPOT_USAGE, depotCiUsage = DEPOT_CI_USAGE } = {}) {
+async function runDigest(
+ t,
+ { env = {}, depotUsage = DEPOT_USAGE, depotCiUsage = DEPOT_CI_USAGE, workflowHealth = WORKFLOW_HEALTH } = {}
+) {
t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: new Date('2026-10-05T13:00:00Z') })
const settings = { ...ENV, ...env }
const previousEnv = Object.fromEntries(Object.keys(settings).map((key) => [key, process.env[key]]))
@@ -73,6 +92,10 @@ async function runDigest(t, { env = {}, depotUsage = DEPOT_USAGE, depotCiUsage =
const usage = depotCiUsage[parsed.searchParams.get('date_from')]
return usage instanceof Response ? usage : Response.json(usage)
}
+ if (parsed.pathname.endsWith('/workflow_health/')) {
+ const health = workflowHealth[parsed.searchParams.get('date_from')]
+ return health instanceof Response ? health : Response.json(health)
+ }
return Response.json(OVERVIEW)
}
if (href.startsWith('https://api.depot.dev/')) {
@@ -144,6 +167,36 @@ describe('weekly engineering analytics digest', () => {
blocks[3].text.text,
/runs out around 2026-11-07, 55 days before the contract ends on 2027-01-01\.$/
)
+ const healthUrls = requests
+ .filter((request) => request.url.includes('/workflow_health/'))
+ .map((request) => new URL(request.url))
+ assert.deepEqual(
+ healthUrls.map((url) => url.searchParams.get('run_scope')),
+ ['pull_request', 'pull_request']
+ )
+ assert.equal(blocks[4].text.text, SLOW_WORKFLOWS_NOTE)
+ })
+ }
+
+ for (const [name, workflowHealth, expected] of [
+ [
+ 'every PR workflow is under the budget',
+ { [WEEK]: [], [PRIOR_WEEK]: [] },
+ ['Every PR workflow had a p95 under 20m.'],
+ ],
+ [
+ 'the durations cannot be read',
+ { ...WORKFLOW_HEALTH, [PRIOR_WEEK]: Response.json({ detail: 'Not found.' }, { status: 404 }) },
+ [],
+ ],
+ ]) {
+ it(`reports PR workflow durations when ${name}`, async (t) => {
+ const { blocks } = await runDigest(t, { workflowHealth })
+ assert.deepEqual(
+ blocks.slice(4).map((block) => block.text.text),
+ expected
+ )
+ assert.equal(tableText(blocks)[0][0], 'Depot spend')
})
}
@@ -204,12 +257,32 @@ describe('weekly engineering analytics digest', () => {
tableText(blocks).map((row) => row[0]),
['merged PRs', 'ready→merge median']
)
- assert.equal(blocks.length, 2)
+ assert.deepEqual(
+ blocks.slice(2).map((block) => block.text.text),
+ [SLOW_WORKFLOWS_NOTE]
+ )
assert.doesNotMatch(JSON.stringify(logs), /confidential-response/)
assert.ok(logs.length > 0)
})
}
+ it('splits a long list of slow workflows across sections that Slack accepts', async (t) => {
+ const names = Array.from({ length: 40 }, (_, i) => `Slow workflow ${i}`)
+ const lastWeek = names.map((name) => ({
+ repo: REPO,
+ workflow_name: name,
+ percentile_run_count: 500,
+ p95_seconds: 1800,
+ }))
+ const { blocks } = await runDigest(t, {
+ workflowHealth: { [WEEK]: lastWeek, [PRIOR_WEEK]: [] },
+ })
+ const notes = blocks.slice(4).map((block) => block.text.text)
+ assert.ok(notes.length > 1)
+ assert.ok(notes.every((note) => note.length <= 3000))
+ assert.equal(notes.join('\n').match(/^• /gm).length, names.length)
+ })
+
it('hides spend and never posts to Slack in a dry run', async (t) => {
const { blocks, logs, requests } = await runDigest(t, { env: { DRY_RUN: 'true' } })
assert.equal(blocks, undefined)
diff --git a/.github/workflows/eng-analytics-weekly-digest.yml b/.github/workflows/eng-analytics-weekly-digest.yml
index 8ecc9d23f8a9..2bd6f23bb40d 100644
--- a/.github/workflows/eng-analytics-weekly-digest.yml
+++ b/.github/workflows/eng-analytics-weekly-digest.yml
@@ -1,6 +1,6 @@
name: Engineering analytics weekly digest
-# Posts last week's Depot spend and pull request figures to #alerts-devex every Monday.
+# Posts last week's Depot spend, pull request figures and slow PR workflows to #alerts-devex every Monday.
# .github/scripts/eng-analytics-weekly-digest.mjs says where each figure comes from.
#
# Required. Until these are set, the job logs a warning and posts nothing: