:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(ci): read depot ci spend from depot getusage (#130110)
Georges-Antoine Assie642e4274555
diff --git a/.github/scripts/eng-analytics-weekly-digest.mjs b/.github/scripts/eng-analytics-weekly-digest.mjsindex fcbf83321622..3fa0a577310e 100644--- a/.github/scripts/eng-analytics-weekly-digest.mjs+++ b/.github/scripts/eng-analytics-weekly-digest.mjs@@ -1,7 +1,6 @@// Weekly CI digest for #alerts-devex. It compares the last 7 complete UTC days with the 7 days before.-// Spend covers every repository in the organization. Runner and container build minutes come from-// Depot's usage API. Depot CI comes from the depot_ci_usage endpoint, which counts only repositories-// with a synced Depot source. Cache storage is not counted. PR workflow p95 durations come from the+// Spend covers every repository in the organization. Runner, Depot CI and container build minutes come+// from Depot's usage API. Cache storage is not counted. PR workflow p95 durations come from the// workflow_health endpoint, which computes them over successful pull request runs only.import { pathToFileURL } from 'node:url'@@ -26,9 +25,12 @@ const DEPOT_CONTRACT_USD = Number(process.env.DEPOT_CONTRACT_USD || 0)const DEPOT_CONTRACT_START = process.env.DEPOT_CONTRACT_START || ''const DEPOT_CONTRACT_END = process.env.DEPOT_CONTRACT_END || ''-// The depot_ci_usage endpoint prices Depot CI.const RUNNER_USD_PER_MINUTE = 0.004const BUILD_USD_PER_MINUTE = 0.04+// Depot bills Depot CI in raw vCPU-minutes, while runner minutes are 2-vCPU-equivalent, so the two+// rates are not comparable. Keep this equal to DEPOT_CI_RATE_USD_PER_VCPU_SECOND × 60 in+// products/engineering_analytics/backend/logic/cost.py, so the digest and the app agree.+const DEPOT_CI_USD_PER_VCPU_MINUTE = 0.003const DAY_MS = 24 * 60 * 60 * 1000const WEEK_MS = 7 * DAY_MS@@ -39,13 +41,13 @@ const MIN_PERCENTILE_RUNS = 100// Slack rejects a message with a section text over 3,000 characters.const SLACK_SECTION_MAX_CHARS = 3000-function billedMinutes(entry) {+function billedMinutes(minutes) {// Protobuf JSON omits a field at its zero default.- const minutes = entry?.minutesBilled ?? 0- if (!Number.isFinite(minutes) || minutes < 0) {+ const value = minutes ?? 0+ if (!Number.isFinite(value) || value < 0) {throw new Error('Depot GetUsage returned invalid billed minutes')}- return minutes+ return value}async function depotUsageSpend(startAt, endAt) {@@ -68,36 +70,42 @@ async function depotUsageSpend(startAt, endAt) {}const usage = await res.json().catch(() => null)const jobs = usage?.githubActionsJobs ?? []+ const ciWorkflows = usage?.depotCi ?? []const builds = usage?.containerBuild ?? []- if (!usage || typeof usage !== 'object' || !Array.isArray(jobs) || !Array.isArray(builds)) {+ if (+ !usage ||+ typeof usage !== 'object' ||+ !Array.isArray(jobs) ||+ !Array.isArray(ciWorkflows) ||+ !Array.isArray(builds)+ ) {throw new Error('Depot GetUsage returned invalid usage')}- let runners = 0- let repoRunners = 0- for (const repo of jobs) {- const usd = billedMinutes(repo?.total) * RUNNER_USD_PER_MINUTE- runners += usd- if (isThisRepository(repo?.repo)) {- repoRunners += usd+ const byRepository = (entries, usd) => {+ let all = 0+ let here = 0+ for (const entry of entries) {+ const amount = usd(entry)+ all += amount+ if (isThisRepository(entry?.repo)) {+ here += amount+ }}+ return [all, here]}- const containerBuilds = builds.reduce((sum, project) => sum + billedMinutes(project) * BUILD_USD_PER_MINUTE, 0)- return { runners, repoRunners, containerBuilds }-}--async function depotCiSpend(startAt, endAt) {- const usage = await withRetry(() =>- api('depot_ci_usage', { date_from: startAt.toISOString(), date_to: endAt.toISOString() })+ const [runners, repoRunners] = byRepository(+ jobs,+ (repo) => billedMinutes(repo?.total?.minutesBilled) * RUNNER_USD_PER_MINUTE)- if (!Array.isArray(usage) || usage.some((repo) => !Number.isFinite(repo?.estimated_cost_usd))) {- throw new Error('depot_ci_usage returned invalid usage')- }- const sum = (repos) => repos.reduce((total, repo) => total + repo.estimated_cost_usd, 0)- return {- depotCi: sum(usage),- repoDepotCi: sum(usage.filter((repo) => isThisRepository(repo.repo))),- depotCiRepos: usage.map((repo) => repo.repo),- }+ const [depotCi, repoDepotCi] = byRepository(+ ciWorkflows,+ (workflow) => billedMinutes(workflow?.vcpuMinutesBilled) * DEPOT_CI_USD_PER_VCPU_MINUTE+ )+ const containerBuilds = builds.reduce(+ (sum, project) => sum + billedMinutes(project?.minutesBilled) * BUILD_USD_PER_MINUTE,+ 0+ )+ return { runners, repoRunners, depotCi, repoDepotCi, containerBuilds }}function isThisRepository(name) {@@ -105,11 +113,8 @@ function isThisRepository(name) {}async function spend(startAt, endAt) {- const [usage, ci] = await Promise.all([- withRetry(() => depotUsageSpend(startAt, endAt)),- depotCiSpend(startAt, endAt),- ])- return { ...usage, ...ci, total: usage.runners + ci.depotCi + usage.containerBuilds }+ const usage = await withRetry(() => depotUsageSpend(startAt, endAt))+ return { ...usage, total: usage.runners + usage.depotCi + usage.containerBuilds }}function contractDates(weekEnd) {@@ -273,7 +278,7 @@ async function api(action, params) {const fail = (detail) => {throw Object.assign(new Error(`${action} -> ${res.status}: ${detail}`), { retryable })}- // The message never quotes the body, because depot_ci_usage returns billing data.+ // The message never quotes the body, because it reaches the public Actions log.const parsed = await res.json().catch(() => fail('non-JSON response'))if (!res.ok) {fail(typeof parsed?.detail === 'string' ? parsed.detail : 'request failed')@@ -410,23 +415,12 @@ function buildBlocks(weekStart, weekEnd, rows, notes) {}function spendNotes(weeks, weekEnd) {- if (!weeks) {+ if (!weeks?.contract) {return []}- const repos = weeks.lastWeek.depotCiRepos- const notes = [- repos.length- ? `Depot CI counts ${repos.join(', ')}.`- : 'Depot CI counts no repository, because no Depot source is synced.',+ return [+ DRY_RUN ? `Depot contract sentence ${HIDDEN}.` : contractSummary(weeks.contract, weeks.lastWeek.total, weekEnd),]- if (weeks.contract) {- notes.push(- DRY_RUN- ? `Depot contract sentence ${HIDDEN}.`- : contractSummary(weeks.contract, weeks.lastWeek.total, weekEnd)- )- }- return notes}async function postToSlack(blocks) {diff --git a/.github/scripts/eng-analytics-weekly-digest.test.mjs b/.github/scripts/eng-analytics-weekly-digest.test.mjsindex e2cf10139847..a3787bfbce19 100644--- a/.github/scripts/eng-analytics-weekly-digest.test.mjs+++ b/.github/scripts/eng-analytics-weekly-digest.test.mjs@@ -25,33 +25,33 @@ const ENV = {const WEEK = '2026-09-28T00:00:00.000Z'const PRIOR_WEEK = '2026-09-21T00:00:00.000Z'const CONTRACT = '2026-01-01T00:00:00.000Z'-// Runners $600 ($400 in this repository), builds $400, Depot CI $1,100 ($1,000 here): $2,100.-// The week before: runners $200, builds $200, Depot CI $500: $900. The contract so far: $50,000.+// Runners $600 ($400 in this repository), builds $400, Depot CI $1,200 ($900 here): $2,200.+// The week before: runners $200, builds $200, Depot CI $300: $700. The contract so far: $50,000.const DEPOT_USAGE = {[WEEK]: {githubActionsJobs: [{ repo: 'PostHog/posthog', total: { minutesBilled: 100_000 } },{ repo: 'PostHog/other', total: { minutesBilled: 50_000 } },],+ depotCi: [+ { repo: 'PostHog/posthog', workflowPath: 'ci-backend.yml', vcpuMinutesBilled: 200_000 },+ { repo: 'PostHog/posthog', workflowPath: 'ci-frontend.yml', vcpuMinutesBilled: 100_000 },+ { repo: 'PostHog/posthog-private', workflowPath: 'ci-backend.yml', vcpuMinutesBilled: 100_000 },+ { repo: 'PostHog/posthog-private', workflowPath: 'idle.yml' },+ ],containerBuild: [{ minutesBilled: 10_000 }],},[PRIOR_WEEK]: {githubActionsJobs: [{ repo: 'PostHog/posthog', total: { minutesBilled: 50_000 } }],+ depotCi: [{ repo: 'PostHog/posthog', workflowPath: 'ci-backend.yml', vcpuMinutesBilled: 100_000 }],containerBuild: [{ minutesBilled: 5_000 }],},[CONTRACT]: {githubActionsJobs: [{ repo: 'PostHog/posthog', total: { minutesBilled: 10_000_000 } }],+ depotCi: [{ repo: 'PostHog/posthog', workflowPath: 'ci-backend.yml', vcpuMinutesBilled: 2_000_000 }],containerBuild: [{ minutesBilled: 100_000 }],},}-const DEPOT_CI_USAGE = {- [WEEK]: [- { repo: 'posthog/posthog', estimated_cost_usd: 1000 },- { repo: 'posthog/posthog-private', estimated_cost_usd: 100 },- ],- [PRIOR_WEEK]: [{ repo: 'posthog/posthog', estimated_cost_usd: 500 }],- [CONTRACT]: [{ repo: 'posthog/posthog', estimated_cost_usd: 6000 }],-}const REPO = { provider: 'github', owner: 'PostHog', name: 'posthog' }const WORKFLOW_HEALTH = {[WEEK]: [@@ -70,10 +70,7 @@ const SLOW_WORKFLOWS_NOTE = [].join('\n')let importCount = 0-async function runDigest(- t,- { env = {}, depotUsage = DEPOT_USAGE, depotCiUsage = DEPOT_CI_USAGE, workflowHealth = WORKFLOW_HEALTH } = {}-) {+async function runDigest(t, { env = {}, depotUsage = DEPOT_USAGE, workflowHealth = WORKFLOW_HEALTH } = {}) {t.mock.timers.enable({ apis: ['Date', 'setTimeout'], now: new Date('2026-10-05T13:00:00Z') })const settings = { ...ENV, ...env }const previousEnv = Object.fromEntries(Object.keys(settings).map((key) => [key, process.env[key]]))@@ -88,10 +85,6 @@ async function runDigest(requests.push({ url: href, ...options })if (href.startsWith('https://example.com/')) {const parsed = new URL(href)- if (parsed.pathname.endsWith('/depot_ci_usage/')) {- const usage = depotCiUsage[parsed.searchParams.get('date_from')]- return usage instanceof Response ? usage : Response.json(usage)- }if (parsed.pathname.endsWith('/workflow_health/')) {const health = workflowHealth[parsed.searchParams.get('date_from')]return health instanceof Response ? health : Response.json(health)@@ -153,19 +146,18 @@ describe('weekly engineering analytics digest', () => {assert.equal(overviewUrl.searchParams.get('repo'), 'PostHog/posthog')assert.match(blocks[0].text.text, /2026-09-28 to 2026-10-04 UTC/)assert.deepEqual(tableText(blocks), [- ['Depot spend', '$2,100', '$900', '+133.3%'],+ ['Depot spend', '$2,200', '$700', '+214.3%'],['└ runners', '$600', '$200', '+200.0%'],- ['└ Depot CI', '$1,100', '$500', '+120.0%'],+ ['└ Depot CI', '$1,200', '$300', '+300.0%'],['└ container builds', '$400', '$200', '+100.0%'],- ['$ / merged PR', '$14', '$10', '+40.0%'],+ ['$ / merged PR', '$13', '$7', '+82.0%'],['merged PRs', '100', '70', '+42.9%'],['ready→merge median', '1h', '2h', '-50.0%'],])- assert.equal(blocks[2].text.text, 'Depot CI counts posthog/posthog, posthog/posthog-private.')- assert.match(blocks[3].text.text, /\$50,000 of \$60,000 used through 2026-10-04/)+ assert.match(blocks[2].text.text, /\$50,000 of \$60,000 used through 2026-10-04/)assert.match(- blocks[3].text.text,- /runs out around 2026-11-07, 55 days before the contract ends on 2027-01-01\.$/+ blocks[2].text.text,+ /runs out around 2026-11-05, 56 days before the contract ends on 2027-01-01\.$/)const healthUrls = requests.filter((request) => request.url.includes('/workflow_health/'))@@ -174,7 +166,7 @@ describe('weekly engineering analytics digest', () => {healthUrls.map((url) => url.searchParams.get('run_scope')),['pull_request', 'pull_request'])- assert.equal(blocks[4].text.text, SLOW_WORKFLOWS_NOTE)+ assert.equal(blocks[3].text.text, SLOW_WORKFLOWS_NOTE)})}@@ -193,7 +185,7 @@ describe('weekly engineering analytics digest', () => {it(`reports PR workflow durations when ${name}`, async (t) => {const { blocks } = await runDigest(t, { workflowHealth })assert.deepEqual(- blocks.slice(4).map((block) => block.text.text),+ blocks.slice(3).map((block) => block.text.text),expected)assert.equal(tableText(blocks)[0][0], 'Depot spend')@@ -206,7 +198,7 @@ describe('weekly engineering analytics digest', () => {]) {it(`describes ${name}`, async (t) => {const { blocks } = await runDigest(t, { env: { DEPOT_CONTRACT_USD: allowance } })- assert.match(blocks[3].text.text, expected)+ assert.match(blocks[2].text.text, expected)})}@@ -215,10 +207,7 @@ describe('weekly engineering analytics digest', () => {['the contract read fails',{- depotCiUsage: {- ...DEPOT_CI_USAGE,- [CONTRACT]: Response.json({ detail: 'Not found.' }, { status: 404 }),- },+ depotUsage: { ...DEPOT_USAGE, [CONTRACT]: new Response('confidential-response', { status: 404 }) },},],]) {@@ -241,13 +230,8 @@ describe('weekly engineering analytics digest', () => {{ depotUsage: { ...DEPOT_USAGE, [PRIOR_WEEK]: { githubActionsJobs: [{ total: { minutesBilled: -1 } }] } } },],[- 'a non-JSON depot_ci_usage response',- {- depotCiUsage: {- ...DEPOT_CI_USAGE,- [WEEK]: new Response('confidential-response', { status: 400 }),- },- },+ 'a malformed Depot CI section',+ { depotUsage: { ...DEPOT_USAGE, [WEEK]: { ...DEPOT_USAGE[WEEK], depotCi: { confidential: 'response' } } } },],['a missing token', { env: { DEPOT_TOKEN: '' } }],]) {@@ -277,7 +261,7 @@ describe('weekly engineering analytics digest', () => {const { blocks } = await runDigest(t, {workflowHealth: { [WEEK]: lastWeek, [PRIOR_WEEK]: [] },})- const notes = blocks.slice(4).map((block) => block.text.text)+ const notes = blocks.slice(3).map((block) => block.text.text)assert.ok(notes.length > 1)assert.ok(notes.every((note) => note.length <= 3000))assert.equal(notes.join('\n').match(/^• /gm).length, names.length)diff --git a/.github/workflows/eng-analytics-weekly-digest.yml b/.github/workflows/eng-analytics-weekly-digest.ymlindex 2bd6f23bb40d..efacf96476b9 100644--- a/.github/workflows/eng-analytics-weekly-digest.yml+++ b/.github/workflows/eng-analytics-weekly-digest.yml@@ -4,7 +4,7 @@ name: Engineering analytics weekly digest# .github/scripts/eng-analytics-weekly-digest.mjs says where each figure comes from.## Required. Until these are set, the job logs a warning and posts nothing:-# - vars.ENG_ANALYTICS_PROJECT_ID: the PostHog project with the GitHub and Depot warehouse sources.+# - vars.ENG_ANALYTICS_PROJECT_ID: the PostHog project with the GitHub warehouse source.# - secrets.ENG_ANALYTICS_POSTHOG_API_KEY: a personal API key for that project, scoped to# engineering_analytics:read.# - secrets.SLACK_DEVEX_BOT_TOKEN: the DevEx Slack bot.