:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ci): list slow pr workflows in weekly digest (#129522)
Raúl Negrón-Otero4526747d4216
diff --git a/.github/scripts/eng-analytics-weekly-digest.mjs b/.github/scripts/eng-analytics-weekly-digest.mjsindex 8c28c02b43a2..fcbf83321622 100644--- a/.github/scripts/eng-analytics-weekly-digest.mjs+++ b/.github/scripts/eng-analytics-weekly-digest.mjs@@ -1,7 +1,8 @@// Weekly CI digest for #alerts-devex. It compares the last 7 complete UTC days with the 7 days before.// Spend covers every repository in the organization. Runner and container build minutes come from// Depot's usage API. Depot CI comes from the depot_ci_usage endpoint, which counts only repositories-// with a synced Depot source. Cache storage is not counted.+// with a synced Depot source. Cache storage is not counted. PR workflow p95 durations come from the+// workflow_health endpoint, which computes them over successful pull request runs only.import { pathToFileURL } from 'node:url'@@ -32,6 +33,12 @@ const BUILD_USD_PER_MINUTE = 0.04const DAY_MS = 24 * 60 * 60 * 1000const WEEK_MS = 7 * DAY_MS+const PR_DURATION_BUDGET_SECONDS = 20 * 60+// One slow run can move the p95 of a workflow with few runs in the week by minutes.+const MIN_PERCENTILE_RUNS = 100+// Slack rejects a message with a section text over 3,000 characters.+const SLACK_SECTION_MAX_CHARS = 3000+function billedMinutes(entry) {// Protobuf JSON omits a field at its zero default.const minutes = entry?.minutesBilled ?? 0@@ -153,6 +160,78 @@ async function readSpend(weekStart, weekEnd) {return { lastWeek, priorWeek, contract: used == null ? null : { ...contract, used } }}+function prWorkflowHealth(dateFrom, dateTo) {+ return withRetry(() =>+ api('workflow_health', {+ date_from: dateFrom.toISOString(),+ date_to: dateTo.toISOString(),+ run_scope: 'pull_request',+ source_id: SOURCE_ID,+ repo: GITHUB_REPOSITORY,+ })+ )+}++// PR workflows whose p95 is over the budget, slowest first. Null when either week cannot be read.+async function readSlowWorkflows(weekStart, weekEnd) {+ return Promise.all([+ prWorkflowHealth(weekStart, weekEnd),+ prWorkflowHealth(new Date(weekStart.getTime() - WEEK_MS), weekStart),+ ])+ .then(([lastWeek, priorWeek]) => {+ const priorP95 = new Map(priorWeek.map((w) => [w.workflow_name, w.p95_seconds]))+ return lastWeek+ .filter(+ (w) => w.percentile_run_count >= MIN_PERCENTILE_RUNS && w.p95_seconds > PR_DURATION_BUDGET_SECONDS+ )+ .sort((a, b) => b.p95_seconds - a.p95_seconds)+ .map((w) => ({+ name: w.workflow_name,+ url: workflowRunsUrl(w.repo, w.workflow_name),+ p95: w.p95_seconds,+ priorP95: priorP95.get(w.workflow_name),+ }))+ })+ .catch(unavailable('PR workflow durations'))+}++function workflowRunsUrl(repo, workflowName) {+ const path = [repo.owner, repo.name, 'actions', 'workflows', workflowName].map(encodeURIComponent).join('/')+ return `${HOST}/project/${PROJECT_ID}/engineering-analytics/repos/${path}`+}++function slackEscape(text) {+ return text.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')+}++function splitIntoSections(lines) {+ const notes = []+ for (const line of lines) {+ const last = notes.at(-1)+ if (last != null && last.length + 1 + line.length <= SLACK_SECTION_MAX_CHARS) {+ notes[notes.length - 1] = `${last}\n${line}`+ } else {+ notes.push(line)+ }+ }+ return notes+}++function slowWorkflowNotes(slow) {+ if (slow == null) {+ return []+ }+ const budget = fmtLongDuration(PR_DURATION_BUDGET_SECONDS)+ if (slow.length === 0) {+ return [`Every PR workflow had a p95 under ${budget}.`]+ }+ const lines = slow.map(({ name, url, p95, priorP95 }) => {+ const prior = priorP95 ? ` (${fmtLongDuration(priorP95)} the week before)` : ''+ return `• <${url}|${slackEscape(name)}>: ${fmtLongDuration(p95)}${prior}`+ })+ return splitIntoSections([`*PR workflows with a p95 over ${budget}*`, ...lines])+}+function isoDay(date) {return date.toISOString().slice(0, 10)}@@ -379,7 +458,7 @@ export async function main() {const weekEnd = new Date()weekEnd.setUTCHours(0, 0, 0, 0)const weekStart = new Date(weekEnd.getTime() - WEEK_MS)- const [overview, weeks] = await Promise.all([+ const [overview, weeks, slowWorkflows] = await Promise.all([// Every figure carries its value for the 7 days before date_from, so one call covers both weeks.withRetry(() =>api('repo_overview', {@@ -391,12 +470,14 @@ export async function main() {})),readSpend(weekStart, weekEnd),+ readSlowWorkflows(weekStart, weekEnd),])const rows = tableRows(overview, weeks)if (rows.length === 0) {throw new Error('No figures to post. Check the GitHub and Depot sources and DEPOT_TOKEN.')}- const blocks = buildBlocks(weekStart, weekEnd, rows, spendNotes(weeks, weekEnd))+ const notes = [...spendNotes(weeks, weekEnd), ...slowWorkflowNotes(slowWorkflows)]+ const blocks = buildBlocks(weekStart, weekEnd, rows, notes)if (DRY_RUN) {console.info(JSON.stringify(blocks, null, 2))returndiff --git a/.github/scripts/eng-analytics-weekly-digest.test.mjs b/.github/scripts/eng-analytics-weekly-digest.test.mjsindex e0f78a18020f..e2cf10139847 100644--- a/.github/scripts/eng-analytics-weekly-digest.test.mjs+++ b/.github/scripts/eng-analytics-weekly-digest.test.mjs@@ -52,9 +52,28 @@ const DEPOT_CI_USAGE = {[PRIOR_WEEK]: [{ repo: 'posthog/posthog', estimated_cost_usd: 500 }],[CONTRACT]: [{ repo: 'posthog/posthog', estimated_cost_usd: 6000 }],}+const REPO = { provider: 'github', owner: 'PostHog', name: 'posthog' }+const WORKFLOW_HEALTH = {+ [WEEK]: [+ { repo: REPO, workflow_name: 'Frontend CI', percentile_run_count: 500, p95_seconds: 1000 },+ { repo: REPO, workflow_name: 'Storybook', percentile_run_count: 500, p95_seconds: 1716 },+ { repo: REPO, workflow_name: 'E2E <CI>', percentile_run_count: 300, p95_seconds: 2400 },+ { repo: REPO, workflow_name: 'Rarely run', percentile_run_count: 10, p95_seconds: 3000 },+ ],+ [PRIOR_WEEK]: [{ repo: REPO, workflow_name: 'Storybook', percentile_run_count: 500, p95_seconds: 1500 }],+}+const WORKFLOWS_URL = 'https://example.com/project/1/engineering-analytics/repos/PostHog/posthog/actions/workflows'+const SLOW_WORKFLOWS_NOTE = [+ '*PR workflows with a p95 over 20m*',+ `• <${WORKFLOWS_URL}/E2E%20%3CCI%3E|E2E <CI>>: 40m`,+ `• <${WORKFLOWS_URL}/Storybook|Storybook>: 29m (25m the week before)`,+].join('\n')let importCount = 0-async function runDigest(t, { env = {}, depotUsage = DEPOT_USAGE, depotCiUsage = DEPOT_CI_USAGE } = {}) {+async function runDigest(+ t,+ { env = {}, depotUsage = DEPOT_USAGE, depotCiUsage = DEPOT_CI_USAGE, workflowHealth = WORKFLOW_HEALTH } = {}+) {t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: new Date('2026-10-05T13:00:00Z') })const settings = { ...ENV, ...env }const previousEnv = Object.fromEntries(Object.keys(settings).map((key) => [key, process.env[key]]))@@ -73,6 +92,10 @@ async function runDigest(t, { env = {}, depotUsage = DEPOT_USAGE, depotCiUsage =const usage = depotCiUsage[parsed.searchParams.get('date_from')]return usage instanceof Response ? usage : Response.json(usage)}+ if (parsed.pathname.endsWith('/workflow_health/')) {+ const health = workflowHealth[parsed.searchParams.get('date_from')]+ return health instanceof Response ? health : Response.json(health)+ }return Response.json(OVERVIEW)}if (href.startsWith('https://api.depot.dev/')) {@@ -144,6 +167,36 @@ describe('weekly engineering analytics digest', () => {blocks[3].text.text,/runs out around 2026-11-07, 55 days before the contract ends on 2027-01-01\.$/)+ const healthUrls = requests+ .filter((request) => request.url.includes('/workflow_health/'))+ .map((request) => new URL(request.url))+ assert.deepEqual(+ healthUrls.map((url) => url.searchParams.get('run_scope')),+ ['pull_request', 'pull_request']+ )+ assert.equal(blocks[4].text.text, SLOW_WORKFLOWS_NOTE)+ })+ }++ for (const [name, workflowHealth, expected] of [+ [+ 'every PR workflow is under the budget',+ { [WEEK]: [], [PRIOR_WEEK]: [] },+ ['Every PR workflow had a p95 under 20m.'],+ ],+ [+ 'the durations cannot be read',+ { ...WORKFLOW_HEALTH, [PRIOR_WEEK]: Response.json({ detail: 'Not found.' }, { status: 404 }) },+ [],+ ],+ ]) {+ it(`reports PR workflow durations when ${name}`, async (t) => {+ const { blocks } = await runDigest(t, { workflowHealth })+ assert.deepEqual(+ blocks.slice(4).map((block) => block.text.text),+ expected+ )+ assert.equal(tableText(blocks)[0][0], 'Depot spend')})}@@ -204,12 +257,32 @@ describe('weekly engineering analytics digest', () => {tableText(blocks).map((row) => row[0]),['merged PRs', 'ready→merge median'])- assert.equal(blocks.length, 2)+ assert.deepEqual(+ blocks.slice(2).map((block) => block.text.text),+ [SLOW_WORKFLOWS_NOTE]+ )assert.doesNotMatch(JSON.stringify(logs), /confidential-response/)assert.ok(logs.length > 0)})}+ it('splits a long list of slow workflows across sections that Slack accepts', async (t) => {+ const names = Array.from({ length: 40 }, (_, i) => `Slow workflow ${i}`)+ const lastWeek = names.map((name) => ({+ repo: REPO,+ workflow_name: name,+ percentile_run_count: 500,+ p95_seconds: 1800,+ }))+ const { blocks } = await runDigest(t, {+ workflowHealth: { [WEEK]: lastWeek, [PRIOR_WEEK]: [] },+ })+ const notes = blocks.slice(4).map((block) => block.text.text)+ assert.ok(notes.length > 1)+ assert.ok(notes.every((note) => note.length <= 3000))+ assert.equal(notes.join('\n').match(/^• /gm).length, names.length)+ })+it('hides spend and never posts to Slack in a dry run', async (t) => {const { blocks, logs, requests } = await runDigest(t, { env: { DRY_RUN: 'true' } })assert.equal(blocks, undefined)diff --git a/.github/workflows/eng-analytics-weekly-digest.yml b/.github/workflows/eng-analytics-weekly-digest.ymlindex 8ecc9d23f8a9..2bd6f23bb40d 100644--- a/.github/workflows/eng-analytics-weekly-digest.yml+++ b/.github/workflows/eng-analytics-weekly-digest.yml@@ -1,6 +1,6 @@name: Engineering analytics weekly digest-# Posts last week's Depot spend and pull request figures to #alerts-devex every Monday.+# Posts last week's Depot spend, pull request figures and slow PR workflows to #alerts-devex every Monday.# .github/scripts/eng-analytics-weekly-digest.mjs says where each figure comes from.## Required. Until these are set, the job logs a warning and posts nothing: