Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

ci(telemetry): make daily CI percentiles by PR type answerable (#9860)

teoee6b2361d755
diff --git a/.agents/rules/infra/github-workflows.md b/.agents/rules/infra/github-workflows.md
index f193df4e8e..5d651c3076 100644
--- a/.agents/rules/infra/github-workflows.md
+++ b/.agents/rules/infra/github-workflows.md
@@ -17,6 +17,8 @@ steps; hand-edited YAML drifts and fails CI's `cargo x workflows --check`.
`steps` instead of inlining labels, env, or step definitions.
- Change a generated workflow (header `DO NOT EDIT`) by editing its source module,
then run `cargo x workflows` and commit the regenerated YAML with it.
+- New CI scripts are Rust or TypeScript, never Python. A step with no checkout embeds
+ its script with `include_str!` and runs it on the hosted image's Node.
- A few older workflows are still hand-written YAML. When you change one
substantially, port it to the generator instead of editing the YAML.
diff --git a/.github/workflows/export-ci-telemetry.yml b/.github/workflows/export-ci-telemetry.yml
index 66da19d4c9..2bfc508a06 100644
--- a/.github/workflows/export-ci-telemetry.yml
+++ b/.github/workflows/export-ci-telemetry.yml
@@ -6,8 +6,11 @@ name: Export CI telemetry
types:
- completed
workflows:
+ - Android Play Release
+ - Android Release Drift
- Auto Label PRs
- Build Agent Daemon on Tag
+ - Build CEF with codecs
- Build AppImage
- Build Desktop on Tag
- Build Rust CI Image
@@ -31,15 +34,19 @@ name: Export CI telemetry
- Ensure Daytona Snapshot
- Export Datadog observability definitions
- Migrate Macro Database
+ - Native Check Label
+ - PR Privacy Check
- Publish Android to Google Play
- Publish SDK
- Pull Request
- Pulumi Preview on PR
- Push local stack binaries
- Release SDK
+ - Release Label Check
- Reusable Deploy Service
- Reusable Preview Service
- SDK Check
+ - Test macrod update
- Web App Pr Checks
- cargo deny check
- cargo workspace dependency check
@@ -50,11 +57,14 @@ name: Export CI telemetry
- conventions code check
- deploy pulumi service
- iOS App Store Publish
+ - iOS App Store Release
- iOS Preview
+ - iOS Release Drift
- infra code check
permissions:
actions: read
contents: read
+ pull-requests: read
id-token: write
jobs:
export:
@@ -102,8 +112,612 @@ jobs:
if (ids.size >= 512) throw new Error('Grafana exporter cannot reliably report delivery failures for 512+ spans');
core.setOutput('export', data.jobs.length > 0);
if (!data.jobs.length) core.notice('No jobs in this attempt; no execution trace to export');
+ - name: Collect push, job and step events
+ if: always()
+ run: |
+ node --disable-warning=ExperimentalWarning --input-type=module-typescript <<'TELEMETRY_SCRIPT'
+ // Turn one completed workflow run into CI telemetry events, as an OTLP logs file.
+ //
+ // Runs in the privileged exporter without the ingestion token. Everything here is
+ // best effort: an error becomes a warning, and the observed workflow is never
+ // affected. Events carry ids, names of our own jobs and steps, and timings, never
+ // branch names, actors, titles or file paths.
+
+ import { createHash } from 'node:crypto';
+ import { readFileSync, writeFileSync } from 'node:fs';
+
+ const API = 'https://api.github.com';
+ const PR_EVENTS = new Set(['pull_request', 'pull_request_target']);
+ // Runs after a PR closes, so it is never part of waiting for checks.
+ const NOT_A_CHECK = new Set(['Cleanup Preview']);
+ // Experiment branches stay queryable but are flagged out of the real PR numbers.
+ const EXPERIMENT_BRANCH = /^[^/]+\/ci-/;
+ // A warm volume restores the dev shell in about 4 s, an empty one takes 110 s or more.
+ export const DEV_SHELL_STEP = 'Setup Nix dev shell';
+ const COLD_VOLUME_SECONDS = 30;
+ // A start this long after the push was not caused by it: a reopen or leaving draft.
+ const MAX_PUSH_LAG_SECONDS = 600;
+ // Below this a step is noise, and it stays visible in the trace.
+ const MIN_STEP_SECONDS = 1;
+ // Stop before the exporter's own traces start failing for lack of API budget.
+ const MIN_RATE_REMAINING = 100;
+ const MAX_FILE_PAGES = 3;
+ // Roll-up jobs finish last by design, so they never say what held a push up.
+ const ROLL_UP_JOB = /status[ -]check/i;
+ // Loki refuses a line this far behind its stream's newest one.
+ export const MAX_EVENT_AGE_SECONDS = 1800;
+ const SMALL_RUST_CRATES = 2;
+ const MEDIUM_RUST_CRATES = 9;
+ const WEB_SUFFIXES = ['.ts', '.tsx', '.js', '.jsx', '.css', '.html', '.svelte'];
+ const RUST_ROOT_FILES = new Set(['Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml']);
+ const CRATE_DIRECTORIES = new Set(['src', 'tests', 'benches', 'examples', 'migrations']);
+
+ export type Timed = {
+ started_at?: string | null;
+ completed_at?: string | null;
+ conclusion?: string | null;
+ };
+
+ export type Step = Timed & { name: string };
+
+ export type Job = Timed & {
+ id: number;
+ name: string;
+ created_at?: string | null;
+ labels?: string[] | null;
+ steps?: Step[] | null;
+ };
+
+ export type Run = {
+ id: number;
+ name: string;
+ run_attempt: number;
+ event: string;
+ head_sha: string;
+ head_branch?: string | null;
+ status: string;
+ conclusion: string | null;
+ created_at: string;
+ updated_at: string;
+ pull_requests?: { number: number }[] | null;
+ head_repository?: { fork?: boolean } | null;
+ };
+
+ export type PullRequest = { created_at: string; draft?: boolean };
+
+ type Interval = [Date, Date];
+ type Attributes = Record<string, string | number | boolean>;
+
+ export type TelemetryEvent = {
+ body: string;
+ at: Date;
+ attributes: Attributes;
+ ids: [traceId: string, spanId: string];
+ };
+
+ export interface GithubApi {
+ get(path: string, params?: Record<string, string | number>): Promise<any>;
+ calls: number;
+ rateLimit: number | null;
+ rateRemaining: number | null;
+ }
+
+ function parseTime(value: string | null | undefined): Date | null {
+ return value ? new Date(value) : null;
+ }
+
+ function seconds(start: Date, end: Date): number {
+ return Math.trunc((end.getTime() - start.getTime()) / 1000);
+ }
+
+ function maxBy<T>(items: T[], key: (item: T) => number): T {
+ return items.reduce((best, item) => (key(item) > key(best) ? item : best));
+ }
+
+ export class HttpError extends Error {
+ code: number;
+
+ constructor(code: number) {
+ super(`HTTP ${code}`);
+ this.code = code;
+ }
+ }
+
+ /** Counts its own calls and stops before exhausting the shared token budget. */
+ class Github implements GithubApi {
+ repo: string;
+ token: string;
+ calls = 0;
+ rateLimit: number | null = null;
+ rateRemaining: number | null = null;
+
+ constructor(repo: string, token: string) {
+ this.repo = repo;
+ this.token = token;
+ }
+
+ async get(path: string, params: Record<string, string | number> = {}) {
+ if (this.rateRemaining !== null && this.rateRemaining < MIN_RATE_REMAINING) {
+ throw new Error('GitHub API budget is low');
+ }
+ const query = new URLSearchParams(
+ Object.entries(params).map(([key, value]): [string, string] => [key, String(value)])
+ ).toString();
+ this.calls += 1;
+ const response = await fetch(`${API}/repos/${this.repo}/${path}${query ? `?${query}` : ''}`, {
+ headers: {
+ Authorization: `Bearer ${this.token}`,
+ Accept: 'application/vnd.github+json',
+ },
+ signal: AbortSignal.timeout(20_000),
+ });
+ if (!response.ok) throw new HttpError(response.status);
+ this.rateLimit = Number(response.headers.get('x-ratelimit-limit')) || null;
+ this.rateRemaining = Number(response.headers.get('x-ratelimit-remaining')) || 0;
+ return response.json();
+ }
+ }
+
+ /** The directory holding a crate's sources, which may be nested in a workspace. */
+ function crateRoot(parts: string[]): string {
+ for (const [index, part] of parts.slice(0, -1).entries()) {
+ if (CRATE_DIRECTORIES.has(part)) return parts.slice(0, index).join('/');
+ }
+ return parts.slice(0, -1).join('/') || parts[0];
+ }
+
+ /**
+ * Return [pr_type, rust_crates_changed] from the changed paths.
+ *
+ * Sizes count changed crates, because the workflow API can't see how many packages
+ * path-check selected.
+ */
+ export function classifyFiles(paths: readonly string[]): [string, number] {
+ const crates = new Set<string>();
+ let rootManifest = false;
+ let web = false;
+ let infra = false;
+ for (const path of paths) {
+ const parts = path.split('/');
+ if (parts[0] === 'infra' || parts[0] === '.github' || path.endsWith('.nix')) {
+ infra = true;
+ } else if (
+ path.endsWith('.rs') ||
+ parts[0] === 'crates' ||
+ parts[0] === '.sqlx' ||
+ RUST_ROOT_FILES.has(parts[parts.length - 1])
+ ) {
+ if (parts.length === 1) rootManifest = true;
+ crates.add(crateRoot(parts));
+ } else if (
+ parts[0] === 'apps' ||
+ parts[0] === 'packages' ||
+ WEB_SUFFIXES.some((suffix) => path.endsWith(suffix))
+ ) {
+ web = true;
+ } else if (parts[0] === 'tooling') {
+ infra = true;
+ }
+ }
+ if (crates.size > 0 && web) return ['mixed', crates.size];
+ if (crates.size > 0) {
+ if (rootManifest || crates.size > MEDIUM_RUST_CRATES) return ['rust-large', crates.size];
+ return [crates.size <= SMALL_RUST_CRATES ? 'rust-small' : 'rust-medium', crates.size];
+ }
+ if (web) return ['web', 0];
+ return [infra ? 'infra' : 'other', 0];
+ }
+
+ function runAttributes(run: Run): Attributes {
+ const attributes: Attributes = {
+ workflow: run.name,
+ run_id: run.id,
+ run_attempt: run.run_attempt,
+ event_name: run.event,
+ head_sha: run.head_sha,
+ experiment: EXPERIMENT_BRANCH.test(run.head_branch ?? ''),
+ rerun: run.run_attempt > 1,
+ };
+ const pullRequest = run.pull_requests?.[0];
+ if (pullRequest) attributes.pr_number = pullRequest.number;
+ return attributes;
+ }
+
+ /** Reproduce gha-otel-export's ids, so an event links to its workflow trace. */
+ export function traceIds(
+ repo: string,
+ run: Pick<Run, 'id' | 'run_attempt'>,
+ suffix?: string
+ ): [string, string] {
+ const prefix = `${repo}${run.id}${run.run_attempt}`;
+ const hash = (input: string) => createHash('sha256').update(input).digest('hex');
+ return [hash(`${prefix}t`).slice(0, 32), hash(`${prefix}${suffix ?? 's'}`).slice(16, 32)];
+ }
+
+ /** Started and finished, not skipped, and with times GitHub didn't copy from an old attempt. */
+ function timed(entry: Timed): Interval | null {
+ const start = parseTime(entry.started_at);
+ const end = parseTime(entry.completed_at);
+ if (entry.conclusion === 'skipped' || !start || !end || end < start) return null;
+ return [start, end];
+ }
+
+ function volumeState(job: Job): [string, number] | null {
+ for (const step of job.steps ?? []) {
+ const times = timed(step);
+ if (step.name === DEV_SHELL_STEP && times) {
+ const duration = seconds(...times);
+ return [duration >= COLD_VOLUME_SECONDS ? 'cold' : 'warm', duration];
+ }
+ }
+ return null;
+ }
+
+ /** Null for a partial rerun's copied job, which starts before it was created. */
+ function queueSeconds(job: Job, times: Interval): number | null {
+ const created = parseTime(job.created_at);
+ if (!created || times[0] < created) return null;
+ return seconds(created, times[0]);
+ }
+
+ /** One event per job that ran, and one per step long enough to matter. */
+ export function jobEvents(repo: string, run: Run, jobs: Job[]): TelemetryEvent[] {
+ const base = runAttributes(run);
+ const events: TelemetryEvent[] = [];
+ for (const job of jobs) {
+ const times = timed(job);
+ if (!times) continue;
+ const attributes: Attributes = {
+ ...base,
+ event: 'job',
+ job: job.name,
+ job_id: job.id,
+ conclusion: job.conclusion ?? '',
+ run_seconds: seconds(...times),
+ };
+ const queue = queueSeconds(job, times);
+ if (queue !== null) attributes.queue_seconds = queue;
+ const label = job.labels?.[0];
+ if (label) attributes.runner_profile = label.split(';')[0];
+ const volume = volumeState(job);
+ if (volume) {
+ attributes.volume_state = volume[0];
+ attributes.dev_shell_seconds = volume[1];
+ }
+ events.push({
+ body: `job ${job.id} ${run.run_attempt}`,
+ at: times[1],
+ attributes,
+ ids: traceIds(repo, run, job.name),
+ });
+ for (const [number, step] of (job.steps ?? []).entries()) {
+ const stepTimes = timed(step);
+ if (!stepTimes || seconds(...stepTimes) < MIN_STEP_SECONDS) continue;
+ events.push({
+ body: `step ${job.id} ${number} ${run.run_attempt}`,
+ at: stepTimes[1],
+ attributes: {
+ ...base,
+ event: 'step',
+ job: job.name,
+ job_id: job.id,
+ step: step.name,
+ conclusion: step.conclusion ?? '',
+ run_seconds: seconds(...stepTimes),
+ },
+ ids: traceIds(repo, run, job.name + step.name),
+ });
+ }
+ }
+ return events;
+ }
+
+ /**
+ * The runs started together with this one, and how many such groups came before.
+ *
+ * A run that starts after everything on the head has finished wasn't part of that
+ * wait: it came from a PR edit, a label or leaving draft.
+ */
+ export function sameTriggerRuns(runs: Run[], run: Pick<Run, 'id'>): [Run[], number] {
+ const created = (entry: Run) => new Date(entry.created_at).getTime();
+ const groups: Run[][] = [];
+ let groupEnd = 0;
+ for (const candidate of [...runs].sort((a, b) => created(a) - created(b))) {
+ const end =
+ candidate.status === 'completed' ? new Date(candidate.updated_at).getTime() : Infinity;
+ const group = groups[groups.length - 1];
+ if (group && created(candidate) <= groupEnd) {
+ group.push(candidate);
+ groupEnd = Math.max(groupEnd, end);
+ } else {
+ groups.push([candidate]);
+ groupEnd = end;
+ }
+ }
+ for (const [index, group] of groups.entries()) {
+ if (group.some((candidate) => candidate.id === run.id)) return [group, index];
+ }
+ return [[], 0];
+ }
+
+ /** A workflow's newest run decides, since a PR edit can correct an earlier failure. */
+ function conclusionOf(runs: Run[]): string {
+ const ordered = [...runs].sort(
+ (a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime() || a.id - b.id
+ );
+ const newest = new Map(ordered.map((run) => [run.name, run]));
+ const conclusions = new Set([...newest.values()].map((run) => run.conclusion));
+ if (['failure', 'timed_out', 'startup_failure'].some((failed) => conclusions.has(failed))) {
+ return 'failure';
+ }
+ return conclusions.has('cancelled') ? 'cancelled' : 'success';
+ }
+
+ /** When the wait began: the push, or opening the PR when that came later. */
+ export function clockStart(
+ firstCreated: Date,
+ pushedAt: Date | null,
+ prCreatedAt: Date | null
+ ): [Date, string] {
+ if (!pushedAt) return [firstCreated, 'first_run'];
+ const start = prCreatedAt && prCreatedAt > pushedAt ? prCreatedAt : pushedAt;
+ if (start > firstCreated || seconds(start, firstCreated) > MAX_PUSH_LAG_SECONDS) {
+ return [firstCreated, 'first_run'];
+ }
+ return [start, start === pushedAt ? 'push' : 'pr_open'];
+ }
+
+ type Finished = { end: Date; times: Interval; job: Job; run: Run };
+
+ /**
+ * The whole push: start to last job, by PR type. Null when no job ran.
+ *
+ * `repeat` counts earlier groups of runs on this head. Those later groups are not
+ * the push's wait, so they are flagged as reruns.
+ */
+ export function pushEvent(
+ repo: string,
+ run: Run,
+ runs: Run[],
+ jobsByRun: Map<number, Job[]>,
+ pushedAt: Date | null,
+ pullRequest: PullRequest | null,
+ files: readonly string[] | null,
+ repeat = 0
+ ): TelemetryEvent | null {
+ const finished: Finished[] = [];
+ for (const candidate of runs) {
+ for (const job of jobsByRun.get(candidate.id) ?? []) {
+ const times = timed(job);
+ if (times) finished.push({ end: times[1], times, job, run: candidate });
+ }
+ }
+ if (finished.length === 0) return null;
+ const firstCreated = new Date(Math.min(...runs.map((c) => new Date(c.created_at).getTime())));
+ const prCreatedAt = pullRequest ? new Date(pullRequest.created_at) : null;
+ const [start, startSource] = clockStart(firstCreated, pushedAt, prCreatedAt);
+ const end = maxBy(finished, (entry) => entry.end.getTime()).end;
+ const notRollUp = finished.filter((entry) => !ROLL_UP_JOB.test(entry.job.name));
+ const work = notRollUp.length > 0 ? notRollUp : finished;
+ const last = maxBy(work, (entry) => entry.end.getTime());
+ const slowest = maxBy(work, (entry) => seconds(...entry.times));
+ const queues = finished
+ .map((entry) => queueSeconds(entry.job, entry.times))
+ .filter((queue) => queue !== null);
+ const [prType, rustCrates] = files !== null ? classifyFiles(files) : ['unknown', 0];
+ const attributes: Attributes = {
+ ...runAttributes(run),
+ event: 'push',
+ pr_type: prType,
+ rust_crates_changed: rustCrates,
+ conclusion: conclusionOf(runs),
+ wall_seconds: seconds(start, end),
+ wall_start: startSource,
+ start_lag_seconds: seconds(start, firstCreated),
+ last_workflow: last.run.name,
+ last_job: last.job.name,
+ last_job_seconds: seconds(...last.times),
+ slowest_job: slowest.job.name,
+ slowest_job_seconds: seconds(...slowest.times),
+ queue_max_seconds: Math.max(0, ...queues),
+ queue_total_seconds: queues.reduce((sum, queue) => sum + queue, 0),
+ workflows: runs.length,
+ jobs: finished.length,
+ cold_volumes: finished.filter((entry) => volumeState(entry.job)?.[0] === 'cold').length,
+ rerun: repeat > 0 || runs.some((candidate) => candidate.run_attempt > 1),
+ };
+ delete attributes.workflow;
+ delete attributes.run_id;
+ delete attributes.run_attempt;
+ if (files !== null) attributes.files_changed = files.length;
+ if (pullRequest) attributes.pr_draft = Boolean(pullRequest.draft);
+ if (run.head_repository?.fork) attributes.fork = true;
+ const attempts = runs.reduce((sum, candidate) => sum + candidate.run_attempt, 0);
+ // The body and time are the same from every exporter that sees the push finish,
+ // so Loki drops the duplicates when two runs complete together.
+ return {
+ body: `push ${run.head_sha} ${repeat} ${attempts}`,
+ at: end,
+ attributes,
+ ids: traceIds(repo, last.run, last.job.name),
+ };
+ }
+
+ async function pushedTime(github: GithubApi, branch: string, headSha: string) {
+ const activity: { after?: string; timestamp: string }[] = await github.get('activity', {
+ ref: `refs/heads/${branch}`,
+ per_page: 30,
+ });
+ const entry = activity.find((candidate) => candidate.after === headSha);
+ return entry ? new Date(entry.timestamp) : null;
+ }
+
+ async function changedFiles(github: GithubApi, number: number) {
+ const paths: string[] = [];
+ for (let page = 1; page <= MAX_FILE_PAGES; page += 1) {
+ const batch: { filename: string }[] = await github.get(`pulls/${number}/files`, {
+ per_page: 100,
+ page,
+ });
+ paths.push(...batch.map((entry) => entry.filename));
+ if (batch.length < 100) break;
+ }
+ return paths;
+ }
+
+ /** The push event once this completion is the last one, otherwise null. */
+ export async function collectPush(
+ github: GithubApi,
+ repo: string,
+ run: Run
+ ): Promise<TelemetryEvent | null> {
+ if (!PR_EVENTS.has(run.event)) return null;
+ const listing: { workflow_runs: Run[] } = await github.get('actions/runs', {
+ head_sha: run.head_sha,
+ per_page: 100,
+ });
+ const checks = listing.workflow_runs.filter(
+ (candidate) => PR_EVENTS.has(candidate.event) && !NOT_A_CHECK.has(candidate.name)
+ );
+ const [runs, repeat] = sameTriggerRuns(checks, run);
+ if (runs.length === 0 || runs.some((candidate) => candidate.status !== 'completed')) {
+ return null;
+ }
+ const jobsByRun = new Map<number, Job[]>();
+ for (const candidate of runs) {
+ jobsByRun.set(
+ candidate.id,
+ candidate.conclusion === 'skipped'
+ ? []
+ : (
+ await github.get(
+ `actions/runs/${candidate.id}/attempts/${candidate.run_attempt}/jobs`,
+ { per_page: 100 }
+ )
+ ).jobs
+ );
+ }
+ // GitHub leaves `pull_requests` empty on some runs, so ask the group when this is one.
+ const head = run.pull_requests?.length
+ ? run
+ : (runs.find((candidate) => candidate.pull_requests?.length) ?? run);
+ const number = head.pull_requests?.[0]?.number;
+ const pullRequest: PullRequest | null = number ? await github.get(`pulls/${number}`) : null;
+ const files = number ? await changedFiles(github, number) : null;
+ const pushedAt =
+ !run.head_repository?.fork && run.head_branch
+ ? await pushedTime(github, run.head_branch, run.head_sha)
+ : null;
+ return pushEvent(repo, head, runs, jobsByRun, pushedAt, pullRequest, files, repeat);
+ }
+
+ function otlpValue(value: string | number | boolean) {
+ if (typeof value === 'boolean') return { boolValue: value };
+ if (typeof value === 'number') return { intValue: String(value) };
+ return { stringValue: value };
+ }
+
+ /** OTLP logs. A record's time is its event's, clamped to what Loki will accept. */
+ export function toOtlp(events: TelemetryEvent[], now: Date) {
+ const oldest = now.getTime() - MAX_EVENT_AGE_SECONDS * 1000;
+ const logRecords = events.map(({ body, at, attributes, ids: [traceId, spanId] }) => ({
+ timeUnixNano: `${Math.trunc(Math.max(at.getTime(), oldest) / 1000)}000000000`,
+ severityText: 'INFO',
+ body: { stringValue: body },
+ traceId,
+ spanId,
+ attributes: Object.entries(attributes).map(([key, value]) => ({
+ key,
+ value: otlpValue(value),
+ })),
+ }));
+ const attributes = [
+ { key: 'service.name', value: { stringValue: 'ci-telemetry' } },
+ { key: 'deployment.environment', value: { stringValue: 'ci' } },
+ ];
+ return {
+ resourceLogs: [
+ {
+ resource: { attributes },
+ scopeLogs: [{ scope: { name: 'ci-telemetry' }, logRecords }],
+ },
+ ],
+ };
+ }
+
+ export async function collect(github: GithubApi, repo: string, run: Run) {
+ const events: TelemetryEvent[] = [];
+ try {
+ const { jobs } = await github.get(
+ `actions/runs/${run.id}/attempts/${run.run_attempt}/jobs`,
+ { per_page: 100 }
+ );
+ events.push(...jobEvents(repo, run, jobs));
+ const push = await collectPush(github, repo, run);
+ if (push) events.push(push);
+ } catch (error) {
+ // A partial export still helps. The class and status can't carry a token or a payload.
+ console.log(`::warning::CI telemetry partly collected: ${describe(error)}`);
+ }
+ return events;
+ }
+
+ function describe(error: unknown): string {
+ if (error instanceof HttpError) return `HttpError ${error.code}`;
+ return error instanceof Error ? error.constructor.name : 'unknown error';
+ }
+
+ function required(name: string): string {
+ const value = process.env[name];
+ if (!value) throw new Error('A required environment variable is missing');
+ return value;
+ }
+
+ async function runCollection() {
+ const started = performance.now();
+ const repo = required('GITHUB_REPOSITORY');
+ const run: Run = JSON.parse(readFileSync(required('GITHUB_EVENT_PATH'), 'utf8')).workflow_run;
+ const github = new Github(repo, required('GITHUB_TOKEN'));
+ const events = await collect(github, repo, run);
+ const now = new Date();
+ const counts: Attributes = {
+ event: 'export',
+ ...runAttributes(run),
+ events: events.length,
+ api_calls: github.calls,
+ collect_seconds: Math.round((performance.now() - started) / 1000),
+ };
+ if (github.rateRemaining !== null) counts.rate_remaining = github.rateRemaining;
+ if (github.rateLimit) counts.rate_limit = github.rateLimit;
+ events.push({
+ body: `export ${run.id} ${run.run_attempt}`,
+ at: now,
+ attributes: counts,
+ ids: traceIds(repo, run),
+ });
+ writeFileSync(required('CI_TELEMETRY_EVENTS'), JSON.stringify(toOtlp(events, now)));
+ console.log(`${events.length} events, ${github.calls} API calls`);
+ }
+
+ /** Entry point, called by the workflow step and never by an import. */
+ export async function main() {
+ try {
+ await runCollection();
+ } catch (error) {
+ console.log(`::warning::CI telemetry not collected: ${describe(error)}`);
+ }
+ }
+
+ await main();
+ TELEMETRY_SCRIPT
+ shell: bash
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ CI_TELEMETRY_EVENTS: ${{ runner.temp }}/ci-telemetry-events.json
+ timeout-minutes: 2
+ continue-on-error: true
- name: Assume CI telemetry role
- if: steps.preflight.outputs.export == 'true'
+ if: always()
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b
with:
role-to-assume: arn:aws:iam::569036502058:role/macro-observability-ci
@@ -111,7 +725,7 @@ jobs:
role-duration-seconds: '900'
- id: ingest
name: Read ingestion credential
- if: steps.preflight.outputs.export == 'true'
+ if: always()
run: |
set -euo pipefail
token=$(aws secretsmanager get-secret-value --region us-east-1 \
@@ -125,6 +739,92 @@ jobs:
echo "::add-mask::$headers"
printf 'headers=%s\n' "$headers" >> "$GITHUB_OUTPUT"
shell: bash
+ - name: Send push, job and step events
+ if: always()
+ run: |
+ node --disable-warning=ExperimentalWarning --input-type=module-typescript <<'TELEMETRY_SCRIPT'
+ // Send the collected CI telemetry to the OTLP logs route. Best effort, never fails the job.
+
+ import { existsSync, readFileSync } from 'node:fs';
+ import { gzipSync } from 'node:zlib';
+
+ const ATTEMPTS = 2;
+ const TIMEOUT_MILLISECONDS = 10_000;
+ const RETRY_DELAY_MILLISECONDS = 2_000;
+
+ /** Read the `Authorization=Bearer%20...` form the credential step writes. */
+ export function authorization(headers: string): [string, string] {
+ const separator = headers.indexOf('=');
+ return [headers.slice(0, separator), decodeURIComponent(headers.slice(separator + 1))];
+ }
+
+ async function send(url: string, body: Buffer, header: [string, string]) {
+ const response = await fetch(url, {
+ method: 'POST',
+ body: gzipSync(body),
+ headers: {
+ 'Content-Type': 'application/json',
+ 'Content-Encoding': 'gzip',
+ [header[0]]: header[1],
+ },
+ signal: AbortSignal.timeout(TIMEOUT_MILLISECONDS),
+ });
+ if (!response.ok) throw new Error(`HTTP ${response.status}`);
+ return response.status;
+ }
+
+ function required(name: string): string {
+ const value = process.env[name];
+ if (!value) throw new Error('A required environment variable is missing');
+ return value;
+ }
+
+ async function sendEvents() {
+ const started = performance.now();
+ const path = required('CI_TELEMETRY_EVENTS');
+ if (!existsSync(path)) {
+ console.log('Nothing to send');
+ return;
+ }
+ const body = readFileSync(path);
+ const header = authorization(required('OTLP_HEADERS'));
+ const url = required('OTLP_LOGS_URL');
+ for (let attempt = 1; attempt <= ATTEMPTS; attempt += 1) {
+ try {
+ const status = await send(url, body, header);
+ const elapsed = ((performance.now() - started) / 1000).toFixed(1);
+ console.log(`Sent ${body.length} bytes, HTTP ${status}, ${elapsed} s`);
+ return;
+ } catch (error) {
+ // Only the class, so neither the header nor a response reaches the log.
+ const reason = error instanceof Error ? error.constructor.name : 'unknown error';
+ console.log(`::warning::CI telemetry send attempt ${attempt} failed: ${reason}`);
+ if (attempt < ATTEMPTS) {
+ await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MILLISECONDS));
+ }
+ }
+ }
+ }
+
+ /** Entry point, called by the workflow step and never by an import. */
+ export async function main() {
+ try {
+ await sendEvents();
+ } catch (error) {
+ const reason = error instanceof Error ? error.constructor.name : 'unknown error';
+ console.log(`::warning::CI telemetry not sent: ${reason}`);
+ }
+ }
+
+ await main();
+ TELEMETRY_SCRIPT
+ shell: bash
+ env:
+ CI_TELEMETRY_EVENTS: ${{ runner.temp }}/ci-telemetry-events.json
+ OTLP_LOGS_URL: https://otlp.macro-internal.com/v1/logs
+ OTLP_HEADERS: ${{ steps.ingest.outputs.headers }}
+ timeout-minutes: 1
+ continue-on-error: true
- id: traces
name: Export workflow, job and step traces
if: steps.preflight.outputs.export == 'true'
diff --git a/infra/stacks/observability/README.md b/infra/stacks/observability/README.md
index d6d8375ae6..e6248e51fd 100644
--- a/infra/stacks/observability/README.md
+++ b/infra/stacks/observability/README.md
@@ -5,8 +5,8 @@
`tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs` defines
this workflow. Run `cargo x workflows` to regenerate
`.github/workflows/export-ci-telemetry.yml`; CI checks it for drift. The JavaScript
-preflight and shell credential reader are embedded from adjacent `scripts/` files
-at generation time, so runtime checkout is unnecessary.
+preflight, TypeScript event steps and shell credential reader are embedded from
+adjacent `scripts/` files at generation time, so runtime checkout is unnecessary.
The generated workflow observes completed workflows on its
explicit name list using Grafana's
@@ -14,9 +14,53 @@ explicit name list using Grafana's
action, pinned to that reviewed commit. It reads the triggering attempt's GitHub
metadata and sends OTLP/HTTP protobuf traces to our existing endpoint. The trace
hierarchy is workflow → jobs → steps, stored in Tempo with
-`service.name=github-actions` and `deployment.environment=ci`. There is no custom
-telemetry exporter, triggering PR checkout, artifact download, raw build-log
-collection, or Datadog export. Individual test cases are outside this integration.
+`service.name=github-actions` and `deployment.environment=ci`. There is no
+triggering PR checkout, artifact download, raw build-log collection, or Datadog
+export. Individual test cases are outside this integration.
+
+### CI events in Loki
+
+Tempo refuses metrics queries over three hours and keeps traces for seven days, so
+it cannot answer daily percentiles. The exporter therefore also writes one log line
+per finished push, job and step to Loki, with `service_name="ci-telemetry"`. They
+keep for 30 days and give exact percentiles. Fields are structured metadata, read
+without a parser; each line's `trace_id` links to the run's trace.
+
+| `event` | Fields |
+| --- | --- |
+| `push` | `pr_type`, `wall_seconds`, `wall_start`, `start_lag_seconds`, `conclusion`, `last_workflow`, `last_job`, `slowest_job`, `queue_max_seconds`, `cold_volumes`, `files_changed`, `rust_crates_changed`, `experiment`, `rerun`, `pr_draft` |
+| `job` | `workflow`, `job`, `conclusion`, `run_seconds`, `queue_seconds`, `runner_profile`, `volume_state`, `dev_shell_seconds` |
+| `step` | `workflow`, `job`, `step`, `conclusion`, `run_seconds` |
+| `export` | The exporter's own cost: `collect_seconds`, `api_calls`, `rate_remaining`, `rate_limit` |
+
+`pr_type` is `web`, `rust-small`, `rust-medium`, `rust-large`, `mixed`, `infra`,
+`other` or `unknown`. Rust size counts changed crates: up to 2, up to 9, then large,
+or any root manifest change. `wall_seconds` runs from the push, or from opening the
+PR when that came later, to the last job. If neither fits, such as a reopen or
+leaving draft, it runs from the first workflow run and `wall_start` says
+`first_run`. A push counts as finished when every workflow it started has finished,
+so a workflow a PR runs must be on the exporter's list; a test enforces this. A run
+that starts after everything on its head has finished came from something else, such
+as a PR edit, a label or leaving draft. It forms its own line with `rerun=true`,
+which also leaves a commit's first full run after draft out of the real PR numbers.
+
+Real PR numbers exclude experiments, reruns and drafts. Metadata makes an ungrouped
+query return one series per combination, so always group:
+
+```logql
+quantile_over_time(0.9, {service_name="ci-telemetry"} | event="push"
+ | experiment="false" | rerun="false" | pr_draft="false" | conclusion="success"
+ | unwrap wall_seconds [1d]) by (pr_type)
+```
+
+Lines never carry branch names, actors, titles or file paths. A line's time and
+body are the same from every exporter that sees it, so Loki drops the duplicate
+when two workflows finish together. A workflow that starts after the rest finished
+produces a second `push` line for that head. `volume_state` is `cold` when the dev
+shell step takes 30 s or more, an inference from step timing.
+
+Collection and sending are best effort and cannot fail the exporter, and neither
+touches the observed PR's jobs. Collection runs before any credential is read.
`/d/macro-ci` provides workflow traces, failed/cancelled jobs and steps, and jobs
running longer than five minutes. Open a trace for its execution waterfall and
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs
index ccbd4f884e..a292ef165e 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs
@@ -1,13 +1,21 @@
-//! Export completed GitHub Actions attempts as workflow/job/step traces.
+//! Export completed GitHub Actions attempts as workflow/job/step traces, and as
+//! push, job and step events for percentiles over days.
use gh_workflow::{
Event, Expression, Job, Level, Permissions, Step, Workflow, WorkflowRun, WorkflowRunType,
};
-// Explicit names avoid recursively observing this exporter. Add new workflows here.
+#[cfg(test)]
+mod test;
+
+// Explicit names avoid recursively observing this exporter. Add new workflows here,
+// and every workflow a PR runs: a push counts as finished when these have finished.
const OBSERVED_WORKFLOWS: &[&str] = &[
+ "Android Play Release",
+ "Android Release Drift",
"Auto Label PRs",
"Build Agent Daemon on Tag",
+ "Build CEF with codecs",
"Build AppImage",
"Build Desktop on Tag",
"Build Rust CI Image",
@@ -31,15 +39,19 @@ const OBSERVED_WORKFLOWS: &[&str] = &[
"Ensure Daytona Snapshot",
"Export Datadog observability definitions",
"Migrate Macro Database",
+ "Native Check Label",
+ "PR Privacy Check",
"Publish Android to Google Play",
"Publish SDK",
"Pull Request",
"Pulumi Preview on PR",
"Push local stack binaries",
"Release SDK",
+ "Release Label Check",
"Reusable Deploy Service",
"Reusable Preview Service",
"SDK Check",
+ "Test macrod update",
"Web App Pr Checks",
"cargo deny check",
"cargo workspace dependency check",
@@ -50,10 +62,13 @@ const OBSERVED_WORKFLOWS: &[&str] = &[
"conventions code check",
"deploy pulumi service",
"iOS App Store Publish",
+ "iOS App Store Release",
"iOS Preview",
+ "iOS Release Drift",
"infra code check",
];
const HAS_JOBS: &str = "steps.preflight.outputs.export == 'true'";
+const EVENTS_FILE: &str = "${{ runner.temp }}/ci-telemetry-events.json";
/// Build the CI telemetry observer without checking out code or downloading artifacts.
pub fn export_ci_telemetry() -> Workflow {
@@ -71,15 +86,24 @@ pub fn export_ci_telemetry() -> Workflow {
.permissions(Permissions {
contents: Some(Level::Read),
actions: Some(Level::Read),
+ pull_requests: Some(Level::Read),
id_token: Some(Level::Write),
..Default::default()
})
.add_job("export", export())
}
+/// Run embedded TypeScript on the hosted image's Node, which strips types itself, so
+/// the exporter needs no checkout, build or dependency.
+fn node_script(source: &str) -> String {
+ format!(
+ "node --disable-warning=ExperimentalWarning --input-type=module-typescript <<'TELEMETRY_SCRIPT'\n{source}\nawait main();\nTELEMETRY_SCRIPT\n"
+ )
+}
+
fn export() -> Job {
Job::default()
- // GitHub's hosted image supplies AWS CLI and Python without a checkout/setup step.
+ // GitHub's hosted image supplies AWS CLI, Python and Node without a checkout/setup step.
.runs_on("ubuntu-latest")
.timeout_minutes(10u32)
.add_step(
@@ -92,6 +116,17 @@ fn export() -> Job {
.id("preflight")
.add_with(("script", include_str!("scripts/ci_telemetry_preflight.js"))),
)
+ // Before any credential exists, so nothing the run produced shares a step with it.
+ .add_step(
+ Step::new("Collect push, job and step events")
+ .run(node_script(include_str!("scripts/ci_telemetry_collect.ts")))
+ .shell("bash")
+ .if_condition(Expression::new("always()"))
+ .continue_on_error(true)
+ .timeout_minutes(2u32)
+ .add_env(("GITHUB_TOKEN", "${{ github.token }}"))
+ .add_env(("CI_TELEMETRY_EVENTS", EVENTS_FILE)),
+ )
.add_step(
Step::new("Assume CI telemetry role")
.uses(
@@ -99,7 +134,7 @@ fn export() -> Job {
"configure-aws-credentials",
"e7f100cf4c008499ea8adda475de1042d6975c7b",
)
- .if_condition(Expression::new(HAS_JOBS))
+ .if_condition(Expression::new("always()"))
.add_with((
"role-to-assume",
"arn:aws:iam::569036502058:role/macro-observability-ci",
@@ -111,9 +146,22 @@ fn export() -> Job {
Step::new("Read ingestion credential")
.run(include_str!("scripts/read_ci_ingestion_credential.sh"))
.id("ingest")
- .if_condition(Expression::new(HAS_JOBS))
+ .if_condition(Expression::new("always()"))
.shell("bash"),
)
+ // Ahead of the traces: one failing must not cost the other, and this one is
+ // the only record of a run the trace exporter refuses.
+ .add_step(
+ Step::new("Send push, job and step events")
+ .run(node_script(include_str!("scripts/ci_telemetry_send.ts")))
+ .shell("bash")
+ .if_condition(Expression::new("always()"))
+ .continue_on_error(true)
+ .timeout_minutes(1u32)
+ .add_env(("CI_TELEMETRY_EVENTS", EVENTS_FILE))
+ .add_env(("OTLP_LOGS_URL", "https://otlp.macro-internal.com/v1/logs"))
+ .add_env(("OTLP_HEADERS", "${{ steps.ingest.outputs.headers }}")),
+ )
.add_step(
Step::new("Export workflow, job and step traces")
.uses(
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry/test.rs
new file mode 100644
index 0000000000..4d81a11a22
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry/test.rs
@@ -0,0 +1,104 @@
+use super::*;
+
+fn step_names() -> Vec<String> {
+ let job = serde_json::to_value(export()).unwrap();
+ job["steps"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .map(|step| step["name"].as_str().unwrap().to_owned())
+ .collect()
+}
+
+/// The collector reads data a run produced, so it must run before any credential.
+#[test]
+fn events_are_collected_before_any_credential_exists() {
+ let names = step_names();
+ let position = |name: &str| names.iter().position(|step| step == name).unwrap();
+ assert!(position("Collect push, job and step events") < position("Assume CI telemetry role"));
+ assert!(position("Read ingestion credential") < position("Send push, job and step events"));
+}
+
+/// The event steps are best effort, and must run when the trace preflight refuses a run.
+#[test]
+fn event_steps_cannot_fail_the_exporter_and_survive_preflight() {
+ let job = serde_json::to_value(export()).unwrap();
+ for step in job["steps"].as_array().unwrap() {
+ let name = step["name"].as_str().unwrap();
+ if name.ends_with("push, job and step events") {
+ assert_eq!(step["continue-on-error"], true, "{name}");
+ assert_eq!(step["if"], "always()", "{name}");
+ }
+ }
+}
+
+#[test]
+fn only_the_send_step_sees_the_ingestion_headers() {
+ let job = serde_json::to_value(export()).unwrap();
+ for step in job["steps"].as_array().unwrap() {
+ if step["name"] == "Collect push, job and step events" {
+ assert!(step["env"]["OTLP_HEADERS"].is_null());
+ }
+ }
+}
+
+/// A push counts as finished once every check has, and only a listed workflow's
+/// completion triggers the exporter, so an unlisted one that finishes last loses the push.
+#[test]
+fn every_pull_request_workflow_is_observed() {
+ let workflows = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../../.github/workflows");
+ let mut missing = Vec::new();
+ for entry in std::fs::read_dir(workflows).unwrap() {
+ let path = entry.unwrap().path();
+ if !matches!(
+ path.extension().and_then(|extension| extension.to_str()),
+ Some("yml" | "yaml")
+ ) {
+ continue;
+ }
+ let workflow: serde_yaml::Value =
+ serde_yaml::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
+ let triggers = &workflow["on"];
+ let on_pull_request = ["pull_request", "pull_request_target"].iter().any(|name| {
+ triggers.get(*name).is_some()
+ || triggers.as_str() == Some(name)
+ || triggers
+ .as_sequence()
+ .is_some_and(|list| list.iter().any(|item| item.as_str() == Some(name)))
+ });
+ let name = workflow["name"].as_str().unwrap_or_default();
+ if on_pull_request && !OBSERVED_WORKFLOWS.contains(&name) {
+ missing.push(name.to_owned());
+ }
+ }
+ assert!(missing.is_empty(), "add to OBSERVED_WORKFLOWS: {missing:?}");
+}
+
+/// Actions would expand this in the embedded source before the shell ever saw it.
+#[test]
+fn embedded_scripts_contain_no_actions_expressions() {
+ for script in [
+ include_str!("../scripts/ci_telemetry_collect.ts"),
+ include_str!("../scripts/ci_telemetry_send.ts"),
+ ] {
+ assert!(!script.contains("${{"));
+ }
+}
+
+#[test]
+fn collector_scenarios() {
+ let output = std::process::Command::new("node")
+ .args(["--test", "ci_telemetry.test.ts"])
+ .current_dir(concat!(
+ env!("CARGO_MANIFEST_DIR"),
+ "/src/workflows/scripts"
+ ))
+ .output()
+ .expect("run CI telemetry scenarios with Node");
+ assert!(
+ output.status.success(),
+ "CI telemetry scenarios failed:\n{}\n{}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry.test.ts b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry.test.ts
new file mode 100644
index 0000000000..aef763bbd3
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry.test.ts
@@ -0,0 +1,420 @@
+// Scenarios for the CI telemetry collector and sender. Run by xtask_workflows' tests.
+
+import assert from 'node:assert/strict';
+import { describe, test } from 'node:test';
+import {
+ classifyFiles,
+ clockStart,
+ collect,
+ collectPush,
+ DEV_SHELL_STEP,
+ type GithubApi,
+ type Job,
+ jobEvents,
+ MAX_EVENT_AGE_SECONDS,
+ type PullRequest,
+ pushEvent,
+ type Run,
+ sameTriggerRuns,
+ type Step,
+ toOtlp,
+ traceIds,
+} from './ci_telemetry_collect.ts';
+import { authorization } from './ci_telemetry_send.ts';
+
+const REPO = 'acme/repo';
+
+function at(minute: number, second = 0) {
+ const pad = (value: number) => String(value).padStart(2, '0');
+ return `2026-01-01T10:${pad(minute)}:${pad(second)}Z`;
+}
+
+function when(minute: number, second = 0) {
+ return new Date(at(minute, second));
+}
+
+function run(overrides: Partial<Run> = {}): Run {
+ return {
+ id: 1,
+ name: 'code check',
+ run_attempt: 1,
+ event: 'pull_request',
+ head_sha: 'abc',
+ head_branch: 'someone/some-change',
+ status: 'completed',
+ conclusion: 'success',
+ created_at: at(0, 10),
+ updated_at: at(5),
+ pull_requests: [{ number: 7 }],
+ ...overrides,
+ };
+}
+
+function job(overrides: Partial<Job> = {}): Job {
+ return {
+ id: 11,
+ name: 'test',
+ conclusion: 'success',
+ created_at: at(0, 40),
+ started_at: at(1),
+ completed_at: at(5),
+ labels: ['namespace-profile-linux-large;overrides.cache-tag=x'],
+ steps: [],
+ ...overrides,
+ };
+}
+
+function step(name: string, start: string, end: string): Step {
+ return { name, started_at: start, completed_at: end, conclusion: 'success' };
+}
+
+function jobsMap(entries: Record<number, Job[]>) {
+ return new Map(Object.entries(entries).map(([id, jobs]) => [Number(id), jobs]));
+}
+
+type PushOptions = {
+ pushed?: Date | null;
+ files?: readonly string[] | null;
+ pullRequest?: PullRequest | null;
+ head?: Run;
+ repeat?: number;
+};
+
+function push(runs: Run[], jobs: Record<number, Job[]>, options: PushOptions = {}) {
+ const event = pushEvent(
+ REPO,
+ options.head ?? runs[0],
+ runs,
+ jobsMap(jobs),
+ options.pushed ?? null,
+ options.pullRequest ?? null,
+ options.files === undefined ? ['crates/a/src/lib.rs'] : options.files,
+ options.repeat ?? 0
+ );
+ assert.ok(event);
+ return event;
+}
+
+describe('classifyFiles', () => {
+ test('types', () => {
+ const cases: [string[], string][] = [
+ [['apps/web/src/a.tsx'], 'web'],
+ [['infra/stacks/a.ts', '.github/workflows/a.yml'], 'infra'],
+ [['docs/a.md'], 'other'],
+ [['crates/a/src/lib.rs', 'apps/web/b.ts'], 'mixed'],
+ [['crates/a/src/lib.rs'], 'rust-small'],
+ [['crates/a/src/lib.rs', 'docs/a.md', 'infra/a.ts'], 'rust-small'],
+ [Array.from({ length: 5 }, (_, i) => `crates/c${i}/src/lib.rs`), 'rust-medium'],
+ [Array.from({ length: 10 }, (_, i) => `crates/c${i}/src/lib.rs`), 'rust-large'],
+ [['Cargo.lock'], 'rust-large'],
+ ];
+ for (const [paths, expected] of cases) {
+ assert.equal(classifyFiles(paths)[0], expected, paths.join());
+ }
+ });
+
+ test('nested crates count separately', () => {
+ const paths = Array.from({ length: 4 }, (_, i) => `crates/client/c${i}/src/lib.rs`);
+ assert.deepEqual(classifyFiles(paths), ['rust-medium', 4]);
+ });
+
+ test('one crate counts once across its directories', () => {
+ const paths = ['crates/a/src/lib.rs', 'crates/a/tests/b.rs', 'crates/a/Cargo.toml'];
+ assert.deepEqual(classifyFiles(paths), ['rust-small', 1]);
+ });
+
+ test('a migration counts as Rust', () => {
+ assert.equal(classifyFiles(['crates/db/migrations/1_a.sql'])[0], 'rust-small');
+ });
+
+ test('xtask Rust counts as Rust, not infra', () => {
+ assert.equal(classifyFiles(['tooling/xtask/src/a.rs'])[0], 'rust-small');
+ });
+});
+
+describe('clockStart', () => {
+ test('the push starts the clock', () => {
+ const dayBefore = new Date(when(0).getTime() - 86_400_000);
+ assert.deepEqual(clockStart(when(0, 10), when(0), dayBefore), [when(0), 'push']);
+ });
+
+ test('opening the PR after the push starts the clock', () => {
+ assert.deepEqual(clockStart(when(5, 10), when(0), when(5)), [when(5), 'pr_open']);
+ });
+
+ test('an old push falls back to the first run', () => {
+ assert.deepEqual(clockStart(when(30), when(0), null), [when(30), 'first_run']);
+ });
+
+ test('an unknown push falls back to the first run', () => {
+ assert.deepEqual(clockStart(when(0), null, when(0)), [when(0), 'first_run']);
+ });
+});
+
+describe('jobEvents', () => {
+ test('a job has queue and volume', () => {
+ const steps = [step(DEV_SHELL_STEP, at(1), at(1, 4)), step('instant', at(1), at(1))];
+ const events = jobEvents(REPO, run(), [job({ steps })]);
+ const { attributes } = events[0];
+ assert.equal(attributes.queue_seconds, 20);
+ assert.equal(attributes.run_seconds, 240);
+ assert.equal(attributes.volume_state, 'warm');
+ assert.equal(attributes.runner_profile, 'namespace-profile-linux-large');
+ assert.deepEqual(
+ events.map((event) => event.attributes.event),
+ ['job', 'step']
+ );
+ });
+
+ test('a slow dev shell means cold', () => {
+ const steps = [step(DEV_SHELL_STEP, at(1), at(3))];
+ assert.equal(jobEvents(REPO, run(), [job({ steps })])[0].attributes.volume_state, 'cold');
+ });
+
+ test("a partial rerun's copy has no queue time", () => {
+ const { attributes } = jobEvents(REPO, run(), [job({ created_at: at(9) })])[0];
+ assert.ok(!('queue_seconds' in attributes));
+ });
+
+ test('a skipped job is left out', () => {
+ assert.deepEqual(jobEvents(REPO, run(), [job({ conclusion: 'skipped' })]), []);
+ });
+
+ test('trace ids match the exporter', () => {
+ // A real run and the id its trace has in Tempo.
+ const [traceId, spanId] = traceIds('macro-inc/macro', { id: 42073661884, run_attempt: 1 });
+ assert.equal(traceId, '6957c75506a4f45cba526240dd419983');
+ assert.equal(spanId.length, 16);
+ });
+});
+
+describe('pushEvent', () => {
+ test('wall is push to last job across workflows', () => {
+ const runs = [run(), run({ id: 2, name: 'web' })];
+ const jobs = { 1: [job({ completed_at: at(5) })], 2: [job({ name: 'build', completed_at: at(7), id: 12 })] };
+ const { at: end, attributes } = push(runs, jobs, { pushed: when(0) });
+ assert.equal(attributes.wall_seconds, 420);
+ assert.equal(attributes.wall_start, 'push');
+ assert.equal(attributes.start_lag_seconds, 10);
+ assert.equal(attributes.last_job, 'build');
+ assert.equal(attributes.last_workflow, 'web');
+ assert.deepEqual(end, when(7));
+ });
+
+ test('a roll-up job ends the wall but is not the last job', () => {
+ const jobs = {
+ 1: [
+ job({ completed_at: at(5) }),
+ job({ name: 'Cloud Storage Status Check', started_at: at(5), completed_at: at(6), id: 12 }),
+ ],
+ };
+ const { attributes } = push([run()], jobs);
+ assert.equal(attributes.last_job, 'test');
+ assert.equal(attributes.wall_seconds, 350);
+ });
+
+ test('the same push gives the same line from either completion', () => {
+ const runs = [run(), run({ id: 2, name: 'web' })];
+ const jobs = { 1: [job()], 2: [job({ name: 'build', id: 12 })] };
+ const first = push(runs, jobs);
+ const second = push(runs, jobs, { head: runs[1] });
+ assert.equal(first.body, second.body);
+ assert.deepEqual(first.at, second.at);
+ });
+
+ test('a rerun is flagged and gets a new line', () => {
+ const first = push([run()], { 1: [job()] });
+ const rerun = push([run({ run_attempt: 2 })], { 1: [job()] });
+ assert.equal(rerun.attributes.rerun, true);
+ assert.notEqual(first.body, rerun.body);
+ });
+
+ test('any failure fails the push', () => {
+ const runs = [run(), run({ id: 2, name: 'web', conclusion: 'failure' })];
+ const { attributes } = push(runs, { 1: [job()], 2: [job({ id: 12 })] });
+ assert.equal(attributes.conclusion, 'failure');
+ });
+
+ test("a workflow's newest run decides the conclusion", () => {
+ const runs = [run({ conclusion: 'failure' }), run({ id: 2, created_at: at(2) })];
+ const { attributes } = push(runs, { 1: [job()], 2: [job({ id: 12 })] });
+ assert.equal(attributes.conclusion, 'success');
+ });
+
+ test('a later trigger on the same head is flagged', () => {
+ const event = push([run()], { 1: [job()] }, { repeat: 1 });
+ assert.equal(event.attributes.rerun, true);
+ assert.notEqual(event.body, push([run()], { 1: [job()] }).body);
+ });
+
+ test('no job ran is no event', () => {
+ assert.equal(pushEvent(REPO, run(), [run()], jobsMap({ 1: [] }), null, null, []), null);
+ });
+
+ test('an experiment branch is flagged and never named', () => {
+ const head = run({ head_branch: 'someone/ci-probe' });
+ const { attributes } = push([head], { 1: [job()] });
+ assert.equal(attributes.experiment, true);
+ assert.ok(!JSON.stringify(attributes).includes('someone'));
+ });
+
+ test('no PR is unknown type', () => {
+ assert.equal(push([run()], { 1: [job()] }, { files: null }).attributes.pr_type, 'unknown');
+ });
+});
+
+describe('sameTriggerRuns', () => {
+ test('overlapping runs are one group', () => {
+ const runs = [run(), run({ id: 2, created_at: at(4, 11), updated_at: at(9) })];
+ assert.deepEqual(sameTriggerRuns(runs, runs[1]), [runs, 0]);
+ });
+
+ test('a run joining late extends the group', () => {
+ const runs = [
+ run(),
+ run({ id: 2, created_at: at(4), updated_at: at(9) }),
+ run({ id: 3, created_at: at(8), updated_at: at(10) }),
+ ];
+ assert.deepEqual(sameTriggerRuns(runs, runs[2]), [runs, 0]);
+ });
+
+ test('an edit after everything finished is its own group', () => {
+ const first = run({ updated_at: at(1) });
+ const later = run({ id: 2, created_at: at(2), updated_at: at(3) });
+ assert.deepEqual(sameTriggerRuns([later, first], first), [[first], 0]);
+ assert.deepEqual(sameTriggerRuns([later, first], later), [[later], 1]);
+ });
+
+ test('a run still going keeps the group open', () => {
+ const first = run({ status: 'in_progress' });
+ const later = run({ id: 2, created_at: at(40) });
+ assert.deepEqual(sameTriggerRuns([first, later], later), [[first, later], 0]);
+ });
+
+ test('a run outside the checks has no group', () => {
+ assert.deepEqual(sameTriggerRuns([run()], run({ id: 9 })), [[], 0]);
+ });
+});
+
+class FakeGithub implements GithubApi {
+ responses: Record<string, unknown>;
+ calls = 0;
+ rateLimit = null;
+ rateRemaining = null;
+
+ constructor(responses: Record<string, unknown>) {
+ this.responses = responses;
+ }
+
+ async get(path: string) {
+ this.calls += 1;
+ const response = this.responses[path];
+ if (response instanceof Error) throw response;
+ return response;
+ }
+}
+
+function responses(...runs: Run[]): Record<string, unknown> {
+ return {
+ 'actions/runs': { workflow_runs: runs },
+ 'actions/runs/1/attempts/1/jobs': { jobs: [job()] },
+ 'actions/runs/2/attempts/1/jobs': { jobs: [job({ name: 'build', id: 12 })] },
+ 'pulls/7': { created_at: at(0), draft: false },
+ 'pulls/7/files': [{ filename: 'apps/web/a.ts' }],
+ activity: [{ after: 'abc', timestamp: at(0, 5) }],
+ };
+}
+
+describe('collectPush', () => {
+ test('waits for every check', async () => {
+ const github = new FakeGithub(
+ responses(run(), run({ id: 2, status: 'in_progress', conclusion: null }))
+ );
+ assert.equal(await collectPush(github, REPO, run()), null);
+ assert.equal(github.calls, 1);
+ });
+
+ test('a finished push is classified', async () => {
+ const github = new FakeGithub(responses(run(), run({ id: 2 })));
+ const event = await collectPush(github, REPO, run());
+ assert.equal(event?.attributes.pr_type, 'web');
+ assert.equal(event?.attributes.wall_start, 'push');
+ });
+
+ test('a run without its PR still finds the PR through its group', async () => {
+ const last = run({ id: 2, pull_requests: [] });
+ const github = new FakeGithub(responses(run(), last));
+ const event = await collectPush(github, REPO, last);
+ assert.equal(event?.attributes.pr_type, 'web');
+ assert.equal(event?.attributes.pr_number, 7);
+ });
+
+ test('a later edit does not hold up the push', async () => {
+ const github = new FakeGithub(
+ responses(run(), run({ id: 2, created_at: at(40), status: 'in_progress' }))
+ );
+ const event = await collectPush(github, REPO, run());
+ assert.equal(event?.attributes.rerun, false);
+ });
+
+ test('cleanup and other events do not block', async () => {
+ const github = new FakeGithub(
+ responses(
+ run(),
+ run({ id: 3, name: 'Cleanup Preview', status: 'in_progress' }),
+ run({ id: 4, event: 'push', status: 'in_progress' })
+ )
+ );
+ assert.notEqual(await collectPush(github, REPO, run()), null);
+ });
+
+ test('a main push has no push event', async () => {
+ const github = new FakeGithub({});
+ assert.equal(await collectPush(github, REPO, run({ event: 'push' })), null);
+ assert.equal(github.calls, 0);
+ });
+
+ test('an error keeps the job events', async () => {
+ const github = new FakeGithub({ ...responses(run()), 'actions/runs': new Error('boom') });
+ const events = await collect(github, REPO, run());
+ assert.deepEqual(
+ events.map((event) => event.attributes.event),
+ ['job']
+ );
+ });
+});
+
+describe('toOtlp', () => {
+ test('records are typed and clamped', () => {
+ const now = when(59);
+ const document = toOtlp(
+ [
+ {
+ body: 'line',
+ at: when(0),
+ attributes: { a: 1, b: 'x', c: true },
+ ids: ['t'.repeat(32), 's'.repeat(16)],
+ },
+ ],
+ now
+ );
+ const record = document.resourceLogs[0].scopeLogs[0].logRecords[0];
+ const oldest = Math.trunc(now.getTime() / 1000) - MAX_EVENT_AGE_SECONDS;
+ assert.equal(record.timeUnixNano, `${oldest}000000000`);
+ assert.deepEqual(record.attributes, [
+ { key: 'a', value: { intValue: '1' } },
+ { key: 'b', value: { stringValue: 'x' } },
+ { key: 'c', value: { boolValue: true } },
+ ]);
+ assert.equal(record.traceId, 't'.repeat(32));
+ });
+});
+
+describe('authorization', () => {
+ test("reads the credential step's header form", () => {
+ assert.deepEqual(authorization('Authorization=Bearer%20a%2Fb'), [
+ 'Authorization',
+ 'Bearer a/b',
+ ]);
+ });
+});
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_collect.ts b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_collect.ts
new file mode 100644
index 0000000000..8a4d3b691e
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_collect.ts
@@ -0,0 +1,591 @@
+// Turn one completed workflow run into CI telemetry events, as an OTLP logs file.
+//
+// Runs in the privileged exporter without the ingestion token. Everything here is
+// best effort: an error becomes a warning, and the observed workflow is never
+// affected. Events carry ids, names of our own jobs and steps, and timings, never
+// branch names, actors, titles or file paths.
+
+import { createHash } from 'node:crypto';
+import { readFileSync, writeFileSync } from 'node:fs';
+
+const API = 'https://api.github.com';
+const PR_EVENTS = new Set(['pull_request', 'pull_request_target']);
+// Runs after a PR closes, so it is never part of waiting for checks.
+const NOT_A_CHECK = new Set(['Cleanup Preview']);
+// Experiment branches stay queryable but are flagged out of the real PR numbers.
+const EXPERIMENT_BRANCH = /^[^/]+\/ci-/;
+// A warm volume restores the dev shell in about 4 s, an empty one takes 110 s or more.
+export const DEV_SHELL_STEP = 'Setup Nix dev shell';
+const COLD_VOLUME_SECONDS = 30;
+// A start this long after the push was not caused by it: a reopen or leaving draft.
+const MAX_PUSH_LAG_SECONDS = 600;
+// Below this a step is noise, and it stays visible in the trace.
+const MIN_STEP_SECONDS = 1;
+// Stop before the exporter's own traces start failing for lack of API budget.
+const MIN_RATE_REMAINING = 100;
+const MAX_FILE_PAGES = 3;
+// Roll-up jobs finish last by design, so they never say what held a push up.
+const ROLL_UP_JOB = /status[ -]check/i;
+// Loki refuses a line this far behind its stream's newest one.
+export const MAX_EVENT_AGE_SECONDS = 1800;
+const SMALL_RUST_CRATES = 2;
+const MEDIUM_RUST_CRATES = 9;
+const WEB_SUFFIXES = ['.ts', '.tsx', '.js', '.jsx', '.css', '.html', '.svelte'];
+const RUST_ROOT_FILES = new Set(['Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml']);
+const CRATE_DIRECTORIES = new Set(['src', 'tests', 'benches', 'examples', 'migrations']);
+
+export type Timed = {
+ started_at?: string | null;
+ completed_at?: string | null;
+ conclusion?: string | null;
+};
+
+export type Step = Timed & { name: string };
+
+export type Job = Timed & {
+ id: number;
+ name: string;
+ created_at?: string | null;
+ labels?: string[] | null;
+ steps?: Step[] | null;
+};
+
+export type Run = {
+ id: number;
+ name: string;
+ run_attempt: number;
+ event: string;
+ head_sha: string;
+ head_branch?: string | null;
+ status: string;
+ conclusion: string | null;
+ created_at: string;
+ updated_at: string;
+ pull_requests?: { number: number }[] | null;
+ head_repository?: { fork?: boolean } | null;
+};
+
+export type PullRequest = { created_at: string; draft?: boolean };
+
+type Interval = [Date, Date];
+type Attributes = Record<string, string | number | boolean>;
+
+export type TelemetryEvent = {
+ body: string;
+ at: Date;
+ attributes: Attributes;
+ ids: [traceId: string, spanId: string];
+};
+
+export interface GithubApi {
+ get(path: string, params?: Record<string, string | number>): Promise<any>;
+ calls: number;
+ rateLimit: number | null;
+ rateRemaining: number | null;
+}
+
+function parseTime(value: string | null | undefined): Date | null {
+ return value ? new Date(value) : null;
+}
+
+function seconds(start: Date, end: Date): number {
+ return Math.trunc((end.getTime() - start.getTime()) / 1000);
+}
+
+function maxBy<T>(items: T[], key: (item: T) => number): T {
+ return items.reduce((best, item) => (key(item) > key(best) ? item : best));
+}
+
+export class HttpError extends Error {
+ code: number;
+
+ constructor(code: number) {
+ super(`HTTP ${code}`);
+ this.code = code;
+ }
+}
+
+/** Counts its own calls and stops before exhausting the shared token budget. */
+class Github implements GithubApi {
+ repo: string;
+ token: string;
+ calls = 0;
+ rateLimit: number | null = null;
+ rateRemaining: number | null = null;
+
+ constructor(repo: string, token: string) {
+ this.repo = repo;
+ this.token = token;
+ }
+
+ async get(path: string, params: Record<string, string | number> = {}) {
+ if (this.rateRemaining !== null && this.rateRemaining < MIN_RATE_REMAINING) {
+ throw new Error('GitHub API budget is low');
+ }
+ const query = new URLSearchParams(
+ Object.entries(params).map(([key, value]): [string, string] => [key, String(value)])
+ ).toString();
+ this.calls += 1;
+ const response = await fetch(`${API}/repos/${this.repo}/${path}${query ? `?${query}` : ''}`, {
+ headers: {
+ Authorization: `Bearer ${this.token}`,
+ Accept: 'application/vnd.github+json',
+ },
+ signal: AbortSignal.timeout(20_000),
+ });
+ if (!response.ok) throw new HttpError(response.status);
+ this.rateLimit = Number(response.headers.get('x-ratelimit-limit')) || null;
+ this.rateRemaining = Number(response.headers.get('x-ratelimit-remaining')) || 0;
+ return response.json();
+ }
+}
+
+/** The directory holding a crate's sources, which may be nested in a workspace. */
+function crateRoot(parts: string[]): string {
+ for (const [index, part] of parts.slice(0, -1).entries()) {
+ if (CRATE_DIRECTORIES.has(part)) return parts.slice(0, index).join('/');
+ }
+ return parts.slice(0, -1).join('/') || parts[0];
+}
+
+/**
+ * Return [pr_type, rust_crates_changed] from the changed paths.
+ *
+ * Sizes count changed crates, because the workflow API can't see how many packages
+ * path-check selected.
+ */
+export function classifyFiles(paths: readonly string[]): [string, number] {
+ const crates = new Set<string>();
+ let rootManifest = false;
+ let web = false;
+ let infra = false;
+ for (const path of paths) {
+ const parts = path.split('/');
+ if (parts[0] === 'infra' || parts[0] === '.github' || path.endsWith('.nix')) {
+ infra = true;
+ } else if (
+ path.endsWith('.rs') ||
+ parts[0] === 'crates' ||
+ parts[0] === '.sqlx' ||
+ RUST_ROOT_FILES.has(parts[parts.length - 1])
+ ) {
+ if (parts.length === 1) rootManifest = true;
+ crates.add(crateRoot(parts));
+ } else if (
+ parts[0] === 'apps' ||
+ parts[0] === 'packages' ||
+ WEB_SUFFIXES.some((suffix) => path.endsWith(suffix))
+ ) {
+ web = true;
+ } else if (parts[0] === 'tooling') {
+ infra = true;
+ }
+ }
+ if (crates.size > 0 && web) return ['mixed', crates.size];
+ if (crates.size > 0) {
+ if (rootManifest || crates.size > MEDIUM_RUST_CRATES) return ['rust-large', crates.size];
+ return [crates.size <= SMALL_RUST_CRATES ? 'rust-small' : 'rust-medium', crates.size];
+ }
+ if (web) return ['web', 0];
+ return [infra ? 'infra' : 'other', 0];
+}
+
+function runAttributes(run: Run): Attributes {
+ const attributes: Attributes = {
+ workflow: run.name,
+ run_id: run.id,
+ run_attempt: run.run_attempt,
+ event_name: run.event,
+ head_sha: run.head_sha,
+ experiment: EXPERIMENT_BRANCH.test(run.head_branch ?? ''),
+ rerun: run.run_attempt > 1,
+ };
+ const pullRequest = run.pull_requests?.[0];
+ if (pullRequest) attributes.pr_number = pullRequest.number;
+ return attributes;
+}
+
+/** Reproduce gha-otel-export's ids, so an event links to its workflow trace. */
+export function traceIds(
+ repo: string,
+ run: Pick<Run, 'id' | 'run_attempt'>,
+ suffix?: string
+): [string, string] {
+ const prefix = `${repo}${run.id}${run.run_attempt}`;
+ const hash = (input: string) => createHash('sha256').update(input).digest('hex');
+ return [hash(`${prefix}t`).slice(0, 32), hash(`${prefix}${suffix ?? 's'}`).slice(16, 32)];
+}
+
+/** Started and finished, not skipped, and with times GitHub didn't copy from an old attempt. */
+function timed(entry: Timed): Interval | null {
+ const start = parseTime(entry.started_at);
+ const end = parseTime(entry.completed_at);
+ if (entry.conclusion === 'skipped' || !start || !end || end < start) return null;
+ return [start, end];
+}
+
+function volumeState(job: Job): [string, number] | null {
+ for (const step of job.steps ?? []) {
+ const times = timed(step);
+ if (step.name === DEV_SHELL_STEP && times) {
+ const duration = seconds(...times);
+ return [duration >= COLD_VOLUME_SECONDS ? 'cold' : 'warm', duration];
+ }
+ }
+ return null;
+}
+
+/** Null for a partial rerun's copied job, which starts before it was created. */
+function queueSeconds(job: Job, times: Interval): number | null {
+ const created = parseTime(job.created_at);
+ if (!created || times[0] < created) return null;
+ return seconds(created, times[0]);
+}
+
+/** One event per job that ran, and one per step long enough to matter. */
+export function jobEvents(repo: string, run: Run, jobs: Job[]): TelemetryEvent[] {
+ const base = runAttributes(run);
+ const events: TelemetryEvent[] = [];
+ for (const job of jobs) {
+ const times = timed(job);
+ if (!times) continue;
+ const attributes: Attributes = {
+ ...base,
+ event: 'job',
+ job: job.name,
+ job_id: job.id,
+ conclusion: job.conclusion ?? '',
+ run_seconds: seconds(...times),
+ };
+ const queue = queueSeconds(job, times);
+ if (queue !== null) attributes.queue_seconds = queue;
+ const label = job.labels?.[0];
+ if (label) attributes.runner_profile = label.split(';')[0];
+ const volume = volumeState(job);
+ if (volume) {
+ attributes.volume_state = volume[0];
+ attributes.dev_shell_seconds = volume[1];
+ }
+ events.push({
+ body: `job ${job.id} ${run.run_attempt}`,
+ at: times[1],
+ attributes,
+ ids: traceIds(repo, run, job.name),
+ });
+ for (const [number, step] of (job.steps ?? []).entries()) {
+ const stepTimes = timed(step);
+ if (!stepTimes || seconds(...stepTimes) < MIN_STEP_SECONDS) continue;
+ events.push({
+ body: `step ${job.id} ${number} ${run.run_attempt}`,
+ at: stepTimes[1],
+ attributes: {
+ ...base,
+ event: 'step',
+ job: job.name,
+ job_id: job.id,
+ step: step.name,
+ conclusion: step.conclusion ?? '',
+ run_seconds: seconds(...stepTimes),
+ },
+ ids: traceIds(repo, run, job.name + step.name),
+ });
+ }
+ }
+ return events;
+}
+
+/**
+ * The runs started together with this one, and how many such groups came before.
+ *
+ * A run that starts after everything on the head has finished wasn't part of that
+ * wait: it came from a PR edit, a label or leaving draft.
+ */
+export function sameTriggerRuns(runs: Run[], run: Pick<Run, 'id'>): [Run[], number] {
+ const created = (entry: Run) => new Date(entry.created_at).getTime();
+ const groups: Run[][] = [];
+ let groupEnd = 0;
+ for (const candidate of [...runs].sort((a, b) => created(a) - created(b))) {
+ const end =
+ candidate.status === 'completed' ? new Date(candidate.updated_at).getTime() : Infinity;
+ const group = groups[groups.length - 1];
+ if (group && created(candidate) <= groupEnd) {
+ group.push(candidate);
+ groupEnd = Math.max(groupEnd, end);
+ } else {
+ groups.push([candidate]);
+ groupEnd = end;
+ }
+ }
+ for (const [index, group] of groups.entries()) {
+ if (group.some((candidate) => candidate.id === run.id)) return [group, index];
+ }
+ return [[], 0];
+}
+
+/** A workflow's newest run decides, since a PR edit can correct an earlier failure. */
+function conclusionOf(runs: Run[]): string {
+ const ordered = [...runs].sort(
+ (a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime() || a.id - b.id
+ );
+ const newest = new Map(ordered.map((run) => [run.name, run]));
+ const conclusions = new Set([...newest.values()].map((run) => run.conclusion));
+ if (['failure', 'timed_out', 'startup_failure'].some((failed) => conclusions.has(failed))) {
+ return 'failure';
+ }
+ return conclusions.has('cancelled') ? 'cancelled' : 'success';
+}
+
+/** When the wait began: the push, or opening the PR when that came later. */
+export function clockStart(
+ firstCreated: Date,
+ pushedAt: Date | null,
+ prCreatedAt: Date | null
+): [Date, string] {
+ if (!pushedAt) return [firstCreated, 'first_run'];
+ const start = prCreatedAt && prCreatedAt > pushedAt ? prCreatedAt : pushedAt;
+ if (start > firstCreated || seconds(start, firstCreated) > MAX_PUSH_LAG_SECONDS) {
+ return [firstCreated, 'first_run'];
+ }
+ return [start, start === pushedAt ? 'push' : 'pr_open'];
+}
+
+type Finished = { end: Date; times: Interval; job: Job; run: Run };
+
+/**
+ * The whole push: start to last job, by PR type. Null when no job ran.
+ *
+ * `repeat` counts earlier groups of runs on this head. Those later groups are not
+ * the push's wait, so they are flagged as reruns.
+ */
+export function pushEvent(
+ repo: string,
+ run: Run,
+ runs: Run[],
+ jobsByRun: Map<number, Job[]>,
+ pushedAt: Date | null,
+ pullRequest: PullRequest | null,
+ files: readonly string[] | null,
+ repeat = 0
+): TelemetryEvent | null {
+ const finished: Finished[] = [];
+ for (const candidate of runs) {
+ for (const job of jobsByRun.get(candidate.id) ?? []) {
+ const times = timed(job);
+ if (times) finished.push({ end: times[1], times, job, run: candidate });
+ }
+ }
+ if (finished.length === 0) return null;
+ const firstCreated = new Date(Math.min(...runs.map((c) => new Date(c.created_at).getTime())));
+ const prCreatedAt = pullRequest ? new Date(pullRequest.created_at) : null;
+ const [start, startSource] = clockStart(firstCreated, pushedAt, prCreatedAt);
+ const end = maxBy(finished, (entry) => entry.end.getTime()).end;
+ const notRollUp = finished.filter((entry) => !ROLL_UP_JOB.test(entry.job.name));
+ const work = notRollUp.length > 0 ? notRollUp : finished;
+ const last = maxBy(work, (entry) => entry.end.getTime());
+ const slowest = maxBy(work, (entry) => seconds(...entry.times));
+ const queues = finished
+ .map((entry) => queueSeconds(entry.job, entry.times))
+ .filter((queue) => queue !== null);
+ const [prType, rustCrates] = files !== null ? classifyFiles(files) : ['unknown', 0];
+ const attributes: Attributes = {
+ ...runAttributes(run),
+ event: 'push',
+ pr_type: prType,
+ rust_crates_changed: rustCrates,
+ conclusion: conclusionOf(runs),
+ wall_seconds: seconds(start, end),
+ wall_start: startSource,
+ start_lag_seconds: seconds(start, firstCreated),
+ last_workflow: last.run.name,
+ last_job: last.job.name,
+ last_job_seconds: seconds(...last.times),
+ slowest_job: slowest.job.name,
+ slowest_job_seconds: seconds(...slowest.times),
+ queue_max_seconds: Math.max(0, ...queues),
+ queue_total_seconds: queues.reduce((sum, queue) => sum + queue, 0),
+ workflows: runs.length,
+ jobs: finished.length,
+ cold_volumes: finished.filter((entry) => volumeState(entry.job)?.[0] === 'cold').length,
+ rerun: repeat > 0 || runs.some((candidate) => candidate.run_attempt > 1),
+ };
+ delete attributes.workflow;
+ delete attributes.run_id;
+ delete attributes.run_attempt;
+ if (files !== null) attributes.files_changed = files.length;
+ if (pullRequest) attributes.pr_draft = Boolean(pullRequest.draft);
+ if (run.head_repository?.fork) attributes.fork = true;
+ const attempts = runs.reduce((sum, candidate) => sum + candidate.run_attempt, 0);
+ // The body and time are the same from every exporter that sees the push finish,
+ // so Loki drops the duplicates when two runs complete together.
+ return {
+ body: `push ${run.head_sha} ${repeat} ${attempts}`,
+ at: end,
+ attributes,
+ ids: traceIds(repo, last.run, last.job.name),
+ };
+}
+
+async function pushedTime(github: GithubApi, branch: string, headSha: string) {
+ const activity: { after?: string; timestamp: string }[] = await github.get('activity', {
+ ref: `refs/heads/${branch}`,
+ per_page: 30,
+ });
+ const entry = activity.find((candidate) => candidate.after === headSha);
+ return entry ? new Date(entry.timestamp) : null;
+}
+
+async function changedFiles(github: GithubApi, number: number) {
+ const paths: string[] = [];
+ for (let page = 1; page <= MAX_FILE_PAGES; page += 1) {
+ const batch: { filename: string }[] = await github.get(`pulls/${number}/files`, {
+ per_page: 100,
+ page,
+ });
+ paths.push(...batch.map((entry) => entry.filename));
+ if (batch.length < 100) break;
+ }
+ return paths;
+}
+
+/** The push event once this completion is the last one, otherwise null. */
+export async function collectPush(
+ github: GithubApi,
+ repo: string,
+ run: Run
+): Promise<TelemetryEvent | null> {
+ if (!PR_EVENTS.has(run.event)) return null;
+ const listing: { workflow_runs: Run[] } = await github.get('actions/runs', {
+ head_sha: run.head_sha,
+ per_page: 100,
+ });
+ const checks = listing.workflow_runs.filter(
+ (candidate) => PR_EVENTS.has(candidate.event) && !NOT_A_CHECK.has(candidate.name)
+ );
+ const [runs, repeat] = sameTriggerRuns(checks, run);
+ if (runs.length === 0 || runs.some((candidate) => candidate.status !== 'completed')) {
+ return null;
+ }
+ const jobsByRun = new Map<number, Job[]>();
+ for (const candidate of runs) {
+ jobsByRun.set(
+ candidate.id,
+ candidate.conclusion === 'skipped'
+ ? []
+ : (
+ await github.get(
+ `actions/runs/${candidate.id}/attempts/${candidate.run_attempt}/jobs`,
+ { per_page: 100 }
+ )
+ ).jobs
+ );
+ }
+ // GitHub leaves `pull_requests` empty on some runs, so ask the group when this is one.
+ const head = run.pull_requests?.length
+ ? run
+ : (runs.find((candidate) => candidate.pull_requests?.length) ?? run);
+ const number = head.pull_requests?.[0]?.number;
+ const pullRequest: PullRequest | null = number ? await github.get(`pulls/${number}`) : null;
+ const files = number ? await changedFiles(github, number) : null;
+ const pushedAt =
+ !run.head_repository?.fork && run.head_branch
+ ? await pushedTime(github, run.head_branch, run.head_sha)
+ : null;
+ return pushEvent(repo, head, runs, jobsByRun, pushedAt, pullRequest, files, repeat);
+}
+
+function otlpValue(value: string | number | boolean) {
+ if (typeof value === 'boolean') return { boolValue: value };
+ if (typeof value === 'number') return { intValue: String(value) };
+ return { stringValue: value };
+}
+
+/** OTLP logs. A record's time is its event's, clamped to what Loki will accept. */
+export function toOtlp(events: TelemetryEvent[], now: Date) {
+ const oldest = now.getTime() - MAX_EVENT_AGE_SECONDS * 1000;
+ const logRecords = events.map(({ body, at, attributes, ids: [traceId, spanId] }) => ({
+ timeUnixNano: `${Math.trunc(Math.max(at.getTime(), oldest) / 1000)}000000000`,
+ severityText: 'INFO',
+ body: { stringValue: body },
+ traceId,
+ spanId,
+ attributes: Object.entries(attributes).map(([key, value]) => ({
+ key,
+ value: otlpValue(value),
+ })),
+ }));
+ const attributes = [
+ { key: 'service.name', value: { stringValue: 'ci-telemetry' } },
+ { key: 'deployment.environment', value: { stringValue: 'ci' } },
+ ];
+ return {
+ resourceLogs: [
+ {
+ resource: { attributes },
+ scopeLogs: [{ scope: { name: 'ci-telemetry' }, logRecords }],
+ },
+ ],
+ };
+}
+
+export async function collect(github: GithubApi, repo: string, run: Run) {
+ const events: TelemetryEvent[] = [];
+ try {
+ const { jobs } = await github.get(
+ `actions/runs/${run.id}/attempts/${run.run_attempt}/jobs`,
+ { per_page: 100 }
+ );
+ events.push(...jobEvents(repo, run, jobs));
+ const push = await collectPush(github, repo, run);
+ if (push) events.push(push);
+ } catch (error) {
+ // A partial export still helps. The class and status can't carry a token or a payload.
+ console.log(`::warning::CI telemetry partly collected: ${describe(error)}`);
+ }
+ return events;
+}
+
+function describe(error: unknown): string {
+ if (error instanceof HttpError) return `HttpError ${error.code}`;
+ return error instanceof Error ? error.constructor.name : 'unknown error';
+}
+
+function required(name: string): string {
+ const value = process.env[name];
+ if (!value) throw new Error('A required environment variable is missing');
+ return value;
+}
+
+async function runCollection() {
+ const started = performance.now();
+ const repo = required('GITHUB_REPOSITORY');
+ const run: Run = JSON.parse(readFileSync(required('GITHUB_EVENT_PATH'), 'utf8')).workflow_run;
+ const github = new Github(repo, required('GITHUB_TOKEN'));
+ const events = await collect(github, repo, run);
+ const now = new Date();
+ const counts: Attributes = {
+ event: 'export',
+ ...runAttributes(run),
+ events: events.length,
+ api_calls: github.calls,
+ collect_seconds: Math.round((performance.now() - started) / 1000),
+ };
+ if (github.rateRemaining !== null) counts.rate_remaining = github.rateRemaining;
+ if (github.rateLimit) counts.rate_limit = github.rateLimit;
+ events.push({
+ body: `export ${run.id} ${run.run_attempt}`,
+ at: now,
+ attributes: counts,
+ ids: traceIds(repo, run),
+ });
+ writeFileSync(required('CI_TELEMETRY_EVENTS'), JSON.stringify(toOtlp(events, now)));
+ console.log(`${events.length} events, ${github.calls} API calls`);
+}
+
+/** Entry point, called by the workflow step and never by an import. */
+export async function main() {
+ try {
+ await runCollection();
+ } catch (error) {
+ console.log(`::warning::CI telemetry not collected: ${describe(error)}`);
+ }
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_send.ts b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_send.ts
new file mode 100644
index 0000000000..1c2eb03faf
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_send.ts
@@ -0,0 +1,72 @@
+// Send the collected CI telemetry to the OTLP logs route. Best effort, never fails the job.
+
+import { existsSync, readFileSync } from 'node:fs';
+import { gzipSync } from 'node:zlib';
+
+const ATTEMPTS = 2;
+const TIMEOUT_MILLISECONDS = 10_000;
+const RETRY_DELAY_MILLISECONDS = 2_000;
+
+/** Read the `Authorization=Bearer%20...` form the credential step writes. */
+export function authorization(headers: string): [string, string] {
+ const separator = headers.indexOf('=');
+ return [headers.slice(0, separator), decodeURIComponent(headers.slice(separator + 1))];
+}
+
+async function send(url: string, body: Buffer, header: [string, string]) {
+ const response = await fetch(url, {
+ method: 'POST',
+ body: gzipSync(body),
+ headers: {
+ 'Content-Type': 'application/json',
+ 'Content-Encoding': 'gzip',
+ [header[0]]: header[1],
+ },
+ signal: AbortSignal.timeout(TIMEOUT_MILLISECONDS),
+ });
+ if (!response.ok) throw new Error(`HTTP ${response.status}`);
+ return response.status;
+}
+
+function required(name: string): string {
+ const value = process.env[name];
+ if (!value) throw new Error('A required environment variable is missing');
+ return value;
+}
+
+async function sendEvents() {
+ const started = performance.now();
+ const path = required('CI_TELEMETRY_EVENTS');
+ if (!existsSync(path)) {
+ console.log('Nothing to send');
+ return;
+ }
+ const body = readFileSync(path);
+ const header = authorization(required('OTLP_HEADERS'));
+ const url = required('OTLP_LOGS_URL');
+ for (let attempt = 1; attempt <= ATTEMPTS; attempt += 1) {
+ try {
+ const status = await send(url, body, header);
+ const elapsed = ((performance.now() - started) / 1000).toFixed(1);
+ console.log(`Sent ${body.length} bytes, HTTP ${status}, ${elapsed} s`);
+ return;
+ } catch (error) {
+ // Only the class, so neither the header nor a response reaches the log.
+ const reason = error instanceof Error ? error.constructor.name : 'unknown error';
+ console.log(`::warning::CI telemetry send attempt ${attempt} failed: ${reason}`);
+ if (attempt < ATTEMPTS) {
+ await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MILLISECONDS));
+ }
+ }
+ }
+}
+
+/** Entry point, called by the workflow step and never by an import. */
+export async function main() {
+ try {
+ await sendEvents();
+ } catch (error) {
+ const reason = error instanceof Error ? error.constructor.name : 'unknown error';
+ console.log(`::warning::CI telemetry not sent: ${reason}`);
+ }
+}