Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
ci(telemetry): make daily CI percentiles by PR type answerable (#9860)
teoee6b2361d755
diff --git a/.agents/rules/infra/github-workflows.md b/.agents/rules/infra/github-workflows.mdindex f193df4e8e..5d651c3076 100644--- a/.agents/rules/infra/github-workflows.md+++ b/.agents/rules/infra/github-workflows.md@@ -17,6 +17,8 @@ steps; hand-edited YAML drifts and fails CI's `cargo x workflows --check`.`steps` instead of inlining labels, env, or step definitions.- Change a generated workflow (header `DO NOT EDIT`) by editing its source module,then run `cargo x workflows` and commit the regenerated YAML with it.+- New CI scripts are Rust or TypeScript, never Python. A step with no checkout embeds+ its script with `include_str!` and runs it on the hosted image's Node.- A few older workflows are still hand-written YAML. When you change onesubstantially, port it to the generator instead of editing the YAML.diff --git a/.github/workflows/export-ci-telemetry.yml b/.github/workflows/export-ci-telemetry.ymlindex 66da19d4c9..2bfc508a06 100644--- a/.github/workflows/export-ci-telemetry.yml+++ b/.github/workflows/export-ci-telemetry.yml@@ -6,8 +6,11 @@ name: Export CI telemetrytypes:- completedworkflows:+ - Android Play Release+ - Android Release Drift- Auto Label PRs- Build Agent Daemon on Tag+ - Build CEF with codecs- Build AppImage- Build Desktop on Tag- Build Rust CI Image@@ -31,15 +34,19 @@ name: Export CI telemetry- Ensure Daytona Snapshot- Export Datadog observability definitions- Migrate Macro Database+ - Native Check Label+ - PR Privacy Check- Publish Android to Google Play- Publish SDK- Pull Request- Pulumi Preview on PR- Push local stack binaries- Release SDK+ - Release Label Check- Reusable Deploy Service- Reusable Preview Service- SDK Check+ - Test macrod update- Web App Pr Checks- cargo deny check- cargo workspace dependency check@@ -50,11 +57,14 @@ name: Export CI telemetry- conventions code check- deploy pulumi service- iOS App Store Publish+ - iOS App Store Release- iOS Preview+ - iOS Release Drift- infra code checkpermissions:actions: readcontents: read+ pull-requests: readid-token: writejobs:export:@@ -102,8 +112,612 @@ jobs:if (ids.size >= 512) throw new Error('Grafana exporter cannot reliably report delivery failures for 512+ spans');core.setOutput('export', data.jobs.length > 0);if (!data.jobs.length) core.notice('No jobs in this attempt; no execution trace to export');+ - name: Collect push, job and step events+ if: always()+ run: |+ node --disable-warning=ExperimentalWarning --input-type=module-typescript <<'TELEMETRY_SCRIPT'+ // Turn one completed workflow run into CI telemetry events, as an OTLP logs file.+ //+ // Runs in the privileged exporter without the ingestion token. Everything here is+ // best effort: an error becomes a warning, and the observed workflow is never+ // affected. Events carry ids, names of our own jobs and steps, and timings, never+ // branch names, actors, titles or file paths.++ import { createHash } from 'node:crypto';+ import { readFileSync, writeFileSync } from 'node:fs';++ const API = 'https://api.github.com';+ const PR_EVENTS = new Set(['pull_request', 'pull_request_target']);+ // Runs after a PR closes, so it is never part of waiting for checks.+ const NOT_A_CHECK = new Set(['Cleanup Preview']);+ // Experiment branches stay queryable but are flagged out of the real PR numbers.+ const EXPERIMENT_BRANCH = /^[^/]+\/ci-/;+ // A warm volume restores the dev shell in about 4 s, an empty one takes 110 s or more.+ export const DEV_SHELL_STEP = 'Setup Nix dev shell';+ const COLD_VOLUME_SECONDS = 30;+ // A start this long after the push was not caused by it: a reopen or leaving draft.+ const MAX_PUSH_LAG_SECONDS = 600;+ // Below this a step is noise, and it stays visible in the trace.+ const MIN_STEP_SECONDS = 1;+ // Stop before the exporter's own traces start failing for lack of API budget.+ const MIN_RATE_REMAINING = 100;+ const MAX_FILE_PAGES = 3;+ // Roll-up jobs finish last by design, so they never say what held a push up.+ const ROLL_UP_JOB = /status[ -]check/i;+ // Loki refuses a line this far behind its stream's newest one.+ export const MAX_EVENT_AGE_SECONDS = 1800;+ const SMALL_RUST_CRATES = 2;+ const MEDIUM_RUST_CRATES = 9;+ const WEB_SUFFIXES = ['.ts', '.tsx', '.js', '.jsx', '.css', '.html', '.svelte'];+ const RUST_ROOT_FILES = new Set(['Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml']);+ const CRATE_DIRECTORIES = new Set(['src', 'tests', 'benches', 'examples', 'migrations']);++ export type Timed = {+ started_at?: string | null;+ completed_at?: string | null;+ conclusion?: string | null;+ };++ export type Step = Timed & { name: string };++ export type Job = Timed & {+ id: number;+ name: string;+ created_at?: string | null;+ labels?: string[] | null;+ steps?: Step[] | null;+ };++ export type Run = {+ id: number;+ name: string;+ run_attempt: number;+ event: string;+ head_sha: string;+ head_branch?: string | null;+ status: string;+ conclusion: string | null;+ created_at: string;+ updated_at: string;+ pull_requests?: { number: number }[] | null;+ head_repository?: { fork?: boolean } | null;+ };++ export type PullRequest = { created_at: string; draft?: boolean };++ type Interval = [Date, Date];+ type Attributes = Record<string, string | number | boolean>;++ export type TelemetryEvent = {+ body: string;+ at: Date;+ attributes: Attributes;+ ids: [traceId: string, spanId: string];+ };++ export interface GithubApi {+ get(path: string, params?: Record<string, string | number>): Promise<any>;+ calls: number;+ rateLimit: number | null;+ rateRemaining: number | null;+ }++ function parseTime(value: string | null | undefined): Date | null {+ return value ? new Date(value) : null;+ }++ function seconds(start: Date, end: Date): number {+ return Math.trunc((end.getTime() - start.getTime()) / 1000);+ }++ function maxBy<T>(items: T[], key: (item: T) => number): T {+ return items.reduce((best, item) => (key(item) > key(best) ? item : best));+ }++ export class HttpError extends Error {+ code: number;++ constructor(code: number) {+ super(`HTTP ${code}`);+ this.code = code;+ }+ }++ /** Counts its own calls and stops before exhausting the shared token budget. */+ class Github implements GithubApi {+ repo: string;+ token: string;+ calls = 0;+ rateLimit: number | null = null;+ rateRemaining: number | null = null;++ constructor(repo: string, token: string) {+ this.repo = repo;+ this.token = token;+ }++ async get(path: string, params: Record<string, string | number> = {}) {+ if (this.rateRemaining !== null && this.rateRemaining < MIN_RATE_REMAINING) {+ throw new Error('GitHub API budget is low');+ }+ const query = new URLSearchParams(+ Object.entries(params).map(([key, value]): [string, string] => [key, String(value)])+ ).toString();+ this.calls += 1;+ const response = await fetch(`${API}/repos/${this.repo}/${path}${query ? `?${query}` : ''}`, {+ headers: {+ Authorization: `Bearer ${this.token}`,+ Accept: 'application/vnd.github+json',+ },+ signal: AbortSignal.timeout(20_000),+ });+ if (!response.ok) throw new HttpError(response.status);+ this.rateLimit = Number(response.headers.get('x-ratelimit-limit')) || null;+ this.rateRemaining = Number(response.headers.get('x-ratelimit-remaining')) || 0;+ return response.json();+ }+ }++ /** The directory holding a crate's sources, which may be nested in a workspace. */+ function crateRoot(parts: string[]): string {+ for (const [index, part] of parts.slice(0, -1).entries()) {+ if (CRATE_DIRECTORIES.has(part)) return parts.slice(0, index).join('/');+ }+ return parts.slice(0, -1).join('/') || parts[0];+ }++ /**+ * Return [pr_type, rust_crates_changed] from the changed paths.+ *+ * Sizes count changed crates, because the workflow API can't see how many packages+ * path-check selected.+ */+ export function classifyFiles(paths: readonly string[]): [string, number] {+ const crates = new Set<string>();+ let rootManifest = false;+ let web = false;+ let infra = false;+ for (const path of paths) {+ const parts = path.split('/');+ if (parts[0] === 'infra' || parts[0] === '.github' || path.endsWith('.nix')) {+ infra = true;+ } else if (+ path.endsWith('.rs') ||+ parts[0] === 'crates' ||+ parts[0] === '.sqlx' ||+ RUST_ROOT_FILES.has(parts[parts.length - 1])+ ) {+ if (parts.length === 1) rootManifest = true;+ crates.add(crateRoot(parts));+ } else if (+ parts[0] === 'apps' ||+ parts[0] === 'packages' ||+ WEB_SUFFIXES.some((suffix) => path.endsWith(suffix))+ ) {+ web = true;+ } else if (parts[0] === 'tooling') {+ infra = true;+ }+ }+ if (crates.size > 0 && web) return ['mixed', crates.size];+ if (crates.size > 0) {+ if (rootManifest || crates.size > MEDIUM_RUST_CRATES) return ['rust-large', crates.size];+ return [crates.size <= SMALL_RUST_CRATES ? 'rust-small' : 'rust-medium', crates.size];+ }+ if (web) return ['web', 0];+ return [infra ? 'infra' : 'other', 0];+ }++ function runAttributes(run: Run): Attributes {+ const attributes: Attributes = {+ workflow: run.name,+ run_id: run.id,+ run_attempt: run.run_attempt,+ event_name: run.event,+ head_sha: run.head_sha,+ experiment: EXPERIMENT_BRANCH.test(run.head_branch ?? ''),+ rerun: run.run_attempt > 1,+ };+ const pullRequest = run.pull_requests?.[0];+ if (pullRequest) attributes.pr_number = pullRequest.number;+ return attributes;+ }++ /** Reproduce gha-otel-export's ids, so an event links to its workflow trace. */+ export function traceIds(+ repo: string,+ run: Pick<Run, 'id' | 'run_attempt'>,+ suffix?: string+ ): [string, string] {+ const prefix = `${repo}${run.id}${run.run_attempt}`;+ const hash = (input: string) => createHash('sha256').update(input).digest('hex');+ return [hash(`${prefix}t`).slice(0, 32), hash(`${prefix}${suffix ?? 's'}`).slice(16, 32)];+ }++ /** Started and finished, not skipped, and with times GitHub didn't copy from an old attempt. */+ function timed(entry: Timed): Interval | null {+ const start = parseTime(entry.started_at);+ const end = parseTime(entry.completed_at);+ if (entry.conclusion === 'skipped' || !start || !end || end < start) return null;+ return [start, end];+ }++ function volumeState(job: Job): [string, number] | null {+ for (const step of job.steps ?? []) {+ const times = timed(step);+ if (step.name === DEV_SHELL_STEP && times) {+ const duration = seconds(...times);+ return [duration >= COLD_VOLUME_SECONDS ? 'cold' : 'warm', duration];+ }+ }+ return null;+ }++ /** Null for a partial rerun's copied job, which starts before it was created. */+ function queueSeconds(job: Job, times: Interval): number | null {+ const created = parseTime(job.created_at);+ if (!created || times[0] < created) return null;+ return seconds(created, times[0]);+ }++ /** One event per job that ran, and one per step long enough to matter. */+ export function jobEvents(repo: string, run: Run, jobs: Job[]): TelemetryEvent[] {+ const base = runAttributes(run);+ const events: TelemetryEvent[] = [];+ for (const job of jobs) {+ const times = timed(job);+ if (!times) continue;+ const attributes: Attributes = {+ ...base,+ event: 'job',+ job: job.name,+ job_id: job.id,+ conclusion: job.conclusion ?? '',+ run_seconds: seconds(...times),+ };+ const queue = queueSeconds(job, times);+ if (queue !== null) attributes.queue_seconds = queue;+ const label = job.labels?.[0];+ if (label) attributes.runner_profile = label.split(';')[0];+ const volume = volumeState(job);+ if (volume) {+ attributes.volume_state = volume[0];+ attributes.dev_shell_seconds = volume[1];+ }+ events.push({+ body: `job ${job.id} ${run.run_attempt}`,+ at: times[1],+ attributes,+ ids: traceIds(repo, run, job.name),+ });+ for (const [number, step] of (job.steps ?? []).entries()) {+ const stepTimes = timed(step);+ if (!stepTimes || seconds(...stepTimes) < MIN_STEP_SECONDS) continue;+ events.push({+ body: `step ${job.id} ${number} ${run.run_attempt}`,+ at: stepTimes[1],+ attributes: {+ ...base,+ event: 'step',+ job: job.name,+ job_id: job.id,+ step: step.name,+ conclusion: step.conclusion ?? '',+ run_seconds: seconds(...stepTimes),+ },+ ids: traceIds(repo, run, job.name + step.name),+ });+ }+ }+ return events;+ }++ /**+ * The runs started together with this one, and how many such groups came before.+ *+ * A run that starts after everything on the head has finished wasn't part of that+ * wait: it came from a PR edit, a label or leaving draft.+ */+ export function sameTriggerRuns(runs: Run[], run: Pick<Run, 'id'>): [Run[], number] {+ const created = (entry: Run) => new Date(entry.created_at).getTime();+ const groups: Run[][] = [];+ let groupEnd = 0;+ for (const candidate of [...runs].sort((a, b) => created(a) - created(b))) {+ const end =+ candidate.status === 'completed' ? new Date(candidate.updated_at).getTime() : Infinity;+ const group = groups[groups.length - 1];+ if (group && created(candidate) <= groupEnd) {+ group.push(candidate);+ groupEnd = Math.max(groupEnd, end);+ } else {+ groups.push([candidate]);+ groupEnd = end;+ }+ }+ for (const [index, group] of groups.entries()) {+ if (group.some((candidate) => candidate.id === run.id)) return [group, index];+ }+ return [[], 0];+ }++ /** A workflow's newest run decides, since a PR edit can correct an earlier failure. */+ function conclusionOf(runs: Run[]): string {+ const ordered = [...runs].sort(+ (a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime() || a.id - b.id+ );+ const newest = new Map(ordered.map((run) => [run.name, run]));+ const conclusions = new Set([...newest.values()].map((run) => run.conclusion));+ if (['failure', 'timed_out', 'startup_failure'].some((failed) => conclusions.has(failed))) {+ return 'failure';+ }+ return conclusions.has('cancelled') ? 'cancelled' : 'success';+ }++ /** When the wait began: the push, or opening the PR when that came later. */+ export function clockStart(+ firstCreated: Date,+ pushedAt: Date | null,+ prCreatedAt: Date | null+ ): [Date, string] {+ if (!pushedAt) return [firstCreated, 'first_run'];+ const start = prCreatedAt && prCreatedAt > pushedAt ? prCreatedAt : pushedAt;+ if (start > firstCreated || seconds(start, firstCreated) > MAX_PUSH_LAG_SECONDS) {+ return [firstCreated, 'first_run'];+ }+ return [start, start === pushedAt ? 'push' : 'pr_open'];+ }++ type Finished = { end: Date; times: Interval; job: Job; run: Run };++ /**+ * The whole push: start to last job, by PR type. Null when no job ran.+ *+ * `repeat` counts earlier groups of runs on this head. Those later groups are not+ * the push's wait, so they are flagged as reruns.+ */+ export function pushEvent(+ repo: string,+ run: Run,+ runs: Run[],+ jobsByRun: Map<number, Job[]>,+ pushedAt: Date | null,+ pullRequest: PullRequest | null,+ files: readonly string[] | null,+ repeat = 0+ ): TelemetryEvent | null {+ const finished: Finished[] = [];+ for (const candidate of runs) {+ for (const job of jobsByRun.get(candidate.id) ?? []) {+ const times = timed(job);+ if (times) finished.push({ end: times[1], times, job, run: candidate });+ }+ }+ if (finished.length === 0) return null;+ const firstCreated = new Date(Math.min(...runs.map((c) => new Date(c.created_at).getTime())));+ const prCreatedAt = pullRequest ? new Date(pullRequest.created_at) : null;+ const [start, startSource] = clockStart(firstCreated, pushedAt, prCreatedAt);+ const end = maxBy(finished, (entry) => entry.end.getTime()).end;+ const notRollUp = finished.filter((entry) => !ROLL_UP_JOB.test(entry.job.name));+ const work = notRollUp.length > 0 ? notRollUp : finished;+ const last = maxBy(work, (entry) => entry.end.getTime());+ const slowest = maxBy(work, (entry) => seconds(...entry.times));+ const queues = finished+ .map((entry) => queueSeconds(entry.job, entry.times))+ .filter((queue) => queue !== null);+ const [prType, rustCrates] = files !== null ? classifyFiles(files) : ['unknown', 0];+ const attributes: Attributes = {+ ...runAttributes(run),+ event: 'push',+ pr_type: prType,+ rust_crates_changed: rustCrates,+ conclusion: conclusionOf(runs),+ wall_seconds: seconds(start, end),+ wall_start: startSource,+ start_lag_seconds: seconds(start, firstCreated),+ last_workflow: last.run.name,+ last_job: last.job.name,+ last_job_seconds: seconds(...last.times),+ slowest_job: slowest.job.name,+ slowest_job_seconds: seconds(...slowest.times),+ queue_max_seconds: Math.max(0, ...queues),+ queue_total_seconds: queues.reduce((sum, queue) => sum + queue, 0),+ workflows: runs.length,+ jobs: finished.length,+ cold_volumes: finished.filter((entry) => volumeState(entry.job)?.[0] === 'cold').length,+ rerun: repeat > 0 || runs.some((candidate) => candidate.run_attempt > 1),+ };+ delete attributes.workflow;+ delete attributes.run_id;+ delete attributes.run_attempt;+ if (files !== null) attributes.files_changed = files.length;+ if (pullRequest) attributes.pr_draft = Boolean(pullRequest.draft);+ if (run.head_repository?.fork) attributes.fork = true;+ const attempts = runs.reduce((sum, candidate) => sum + candidate.run_attempt, 0);+ // The body and time are the same from every exporter that sees the push finish,+ // so Loki drops the duplicates when two runs complete together.+ return {+ body: `push ${run.head_sha} ${repeat} ${attempts}`,+ at: end,+ attributes,+ ids: traceIds(repo, last.run, last.job.name),+ };+ }++ async function pushedTime(github: GithubApi, branch: string, headSha: string) {+ const activity: { after?: string; timestamp: string }[] = await github.get('activity', {+ ref: `refs/heads/${branch}`,+ per_page: 30,+ });+ const entry = activity.find((candidate) => candidate.after === headSha);+ return entry ? new Date(entry.timestamp) : null;+ }++ async function changedFiles(github: GithubApi, number: number) {+ const paths: string[] = [];+ for (let page = 1; page <= MAX_FILE_PAGES; page += 1) {+ const batch: { filename: string }[] = await github.get(`pulls/${number}/files`, {+ per_page: 100,+ page,+ });+ paths.push(...batch.map((entry) => entry.filename));+ if (batch.length < 100) break;+ }+ return paths;+ }++ /** The push event once this completion is the last one, otherwise null. */+ export async function collectPush(+ github: GithubApi,+ repo: string,+ run: Run+ ): Promise<TelemetryEvent | null> {+ if (!PR_EVENTS.has(run.event)) return null;+ const listing: { workflow_runs: Run[] } = await github.get('actions/runs', {+ head_sha: run.head_sha,+ per_page: 100,+ });+ const checks = listing.workflow_runs.filter(+ (candidate) => PR_EVENTS.has(candidate.event) && !NOT_A_CHECK.has(candidate.name)+ );+ const [runs, repeat] = sameTriggerRuns(checks, run);+ if (runs.length === 0 || runs.some((candidate) => candidate.status !== 'completed')) {+ return null;+ }+ const jobsByRun = new Map<number, Job[]>();+ for (const candidate of runs) {+ jobsByRun.set(+ candidate.id,+ candidate.conclusion === 'skipped'+ ? []+ : (+ await github.get(+ `actions/runs/${candidate.id}/attempts/${candidate.run_attempt}/jobs`,+ { per_page: 100 }+ )+ ).jobs+ );+ }+ // GitHub leaves `pull_requests` empty on some runs, so ask the group when this is one.+ const head = run.pull_requests?.length+ ? run+ : (runs.find((candidate) => candidate.pull_requests?.length) ?? run);+ const number = head.pull_requests?.[0]?.number;+ const pullRequest: PullRequest | null = number ? await github.get(`pulls/${number}`) : null;+ const files = number ? await changedFiles(github, number) : null;+ const pushedAt =+ !run.head_repository?.fork && run.head_branch+ ? await pushedTime(github, run.head_branch, run.head_sha)+ : null;+ return pushEvent(repo, head, runs, jobsByRun, pushedAt, pullRequest, files, repeat);+ }++ function otlpValue(value: string | number | boolean) {+ if (typeof value === 'boolean') return { boolValue: value };+ if (typeof value === 'number') return { intValue: String(value) };+ return { stringValue: value };+ }++ /** OTLP logs. A record's time is its event's, clamped to what Loki will accept. */+ export function toOtlp(events: TelemetryEvent[], now: Date) {+ const oldest = now.getTime() - MAX_EVENT_AGE_SECONDS * 1000;+ const logRecords = events.map(({ body, at, attributes, ids: [traceId, spanId] }) => ({+ timeUnixNano: `${Math.trunc(Math.max(at.getTime(), oldest) / 1000)}000000000`,+ severityText: 'INFO',+ body: { stringValue: body },+ traceId,+ spanId,+ attributes: Object.entries(attributes).map(([key, value]) => ({+ key,+ value: otlpValue(value),+ })),+ }));+ const attributes = [+ { key: 'service.name', value: { stringValue: 'ci-telemetry' } },+ { key: 'deployment.environment', value: { stringValue: 'ci' } },+ ];+ return {+ resourceLogs: [+ {+ resource: { attributes },+ scopeLogs: [{ scope: { name: 'ci-telemetry' }, logRecords }],+ },+ ],+ };+ }++ export async function collect(github: GithubApi, repo: string, run: Run) {+ const events: TelemetryEvent[] = [];+ try {+ const { jobs } = await github.get(+ `actions/runs/${run.id}/attempts/${run.run_attempt}/jobs`,+ { per_page: 100 }+ );+ events.push(...jobEvents(repo, run, jobs));+ const push = await collectPush(github, repo, run);+ if (push) events.push(push);+ } catch (error) {+ // A partial export still helps. The class and status can't carry a token or a payload.+ console.log(`::warning::CI telemetry partly collected: ${describe(error)}`);+ }+ return events;+ }++ function describe(error: unknown): string {+ if (error instanceof HttpError) return `HttpError ${error.code}`;+ return error instanceof Error ? error.constructor.name : 'unknown error';+ }++ function required(name: string): string {+ const value = process.env[name];+ if (!value) throw new Error('A required environment variable is missing');+ return value;+ }++ async function runCollection() {+ const started = performance.now();+ const repo = required('GITHUB_REPOSITORY');+ const run: Run = JSON.parse(readFileSync(required('GITHUB_EVENT_PATH'), 'utf8')).workflow_run;+ const github = new Github(repo, required('GITHUB_TOKEN'));+ const events = await collect(github, repo, run);+ const now = new Date();+ const counts: Attributes = {+ event: 'export',+ ...runAttributes(run),+ events: events.length,+ api_calls: github.calls,+ collect_seconds: Math.round((performance.now() - started) / 1000),+ };+ if (github.rateRemaining !== null) counts.rate_remaining = github.rateRemaining;+ if (github.rateLimit) counts.rate_limit = github.rateLimit;+ events.push({+ body: `export ${run.id} ${run.run_attempt}`,+ at: now,+ attributes: counts,+ ids: traceIds(repo, run),+ });+ writeFileSync(required('CI_TELEMETRY_EVENTS'), JSON.stringify(toOtlp(events, now)));+ console.log(`${events.length} events, ${github.calls} API calls`);+ }++ /** Entry point, called by the workflow step and never by an import. */+ export async function main() {+ try {+ await runCollection();+ } catch (error) {+ console.log(`::warning::CI telemetry not collected: ${describe(error)}`);+ }+ }++ await main();+ TELEMETRY_SCRIPT+ shell: bash+ env:+ GITHUB_TOKEN: ${{ github.token }}+ CI_TELEMETRY_EVENTS: ${{ runner.temp }}/ci-telemetry-events.json+ timeout-minutes: 2+ continue-on-error: true- name: Assume CI telemetry role- if: steps.preflight.outputs.export == 'true'+ if: always()uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7bwith:role-to-assume: arn:aws:iam::569036502058:role/macro-observability-ci@@ -111,7 +725,7 @@ jobs:role-duration-seconds: '900'- id: ingestname: Read ingestion credential- if: steps.preflight.outputs.export == 'true'+ if: always()run: |set -euo pipefailtoken=$(aws secretsmanager get-secret-value --region us-east-1 \@@ -125,6 +739,92 @@ jobs:echo "::add-mask::$headers"printf 'headers=%s\n' "$headers" >> "$GITHUB_OUTPUT"shell: bash+ - name: Send push, job and step events+ if: always()+ run: |+ node --disable-warning=ExperimentalWarning --input-type=module-typescript <<'TELEMETRY_SCRIPT'+ // Send the collected CI telemetry to the OTLP logs route. Best effort, never fails the job.++ import { existsSync, readFileSync } from 'node:fs';+ import { gzipSync } from 'node:zlib';++ const ATTEMPTS = 2;+ const TIMEOUT_MILLISECONDS = 10_000;+ const RETRY_DELAY_MILLISECONDS = 2_000;++ /** Read the `Authorization=Bearer%20...` form the credential step writes. */+ export function authorization(headers: string): [string, string] {+ const separator = headers.indexOf('=');+ return [headers.slice(0, separator), decodeURIComponent(headers.slice(separator + 1))];+ }++ async function send(url: string, body: Buffer, header: [string, string]) {+ const response = await fetch(url, {+ method: 'POST',+ body: gzipSync(body),+ headers: {+ 'Content-Type': 'application/json',+ 'Content-Encoding': 'gzip',+ [header[0]]: header[1],+ },+ signal: AbortSignal.timeout(TIMEOUT_MILLISECONDS),+ });+ if (!response.ok) throw new Error(`HTTP ${response.status}`);+ return response.status;+ }++ function required(name: string): string {+ const value = process.env[name];+ if (!value) throw new Error('A required environment variable is missing');+ return value;+ }++ async function sendEvents() {+ const started = performance.now();+ const path = required('CI_TELEMETRY_EVENTS');+ if (!existsSync(path)) {+ console.log('Nothing to send');+ return;+ }+ const body = readFileSync(path);+ const header = authorization(required('OTLP_HEADERS'));+ const url = required('OTLP_LOGS_URL');+ for (let attempt = 1; attempt <= ATTEMPTS; attempt += 1) {+ try {+ const status = await send(url, body, header);+ const elapsed = ((performance.now() - started) / 1000).toFixed(1);+ console.log(`Sent ${body.length} bytes, HTTP ${status}, ${elapsed} s`);+ return;+ } catch (error) {+ // Only the class, so neither the header nor a response reaches the log.+ const reason = error instanceof Error ? error.constructor.name : 'unknown error';+ console.log(`::warning::CI telemetry send attempt ${attempt} failed: ${reason}`);+ if (attempt < ATTEMPTS) {+ await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MILLISECONDS));+ }+ }+ }+ }++ /** Entry point, called by the workflow step and never by an import. */+ export async function main() {+ try {+ await sendEvents();+ } catch (error) {+ const reason = error instanceof Error ? error.constructor.name : 'unknown error';+ console.log(`::warning::CI telemetry not sent: ${reason}`);+ }+ }++ await main();+ TELEMETRY_SCRIPT+ shell: bash+ env:+ CI_TELEMETRY_EVENTS: ${{ runner.temp }}/ci-telemetry-events.json+ OTLP_LOGS_URL: https://otlp.macro-internal.com/v1/logs+ OTLP_HEADERS: ${{ steps.ingest.outputs.headers }}+ timeout-minutes: 1+ continue-on-error: true- id: tracesname: Export workflow, job and step tracesif: steps.preflight.outputs.export == 'true'diff --git a/infra/stacks/observability/README.md b/infra/stacks/observability/README.mdindex d6d8375ae6..e6248e51fd 100644--- a/infra/stacks/observability/README.md+++ b/infra/stacks/observability/README.md@@ -5,8 +5,8 @@`tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs` definesthis workflow. Run `cargo x workflows` to regenerate`.github/workflows/export-ci-telemetry.yml`; CI checks it for drift. The JavaScript-preflight and shell credential reader are embedded from adjacent `scripts/` files-at generation time, so runtime checkout is unnecessary.+preflight, TypeScript event steps and shell credential reader are embedded from+adjacent `scripts/` files at generation time, so runtime checkout is unnecessary.The generated workflow observes completed workflows on itsexplicit name list using Grafana's@@ -14,9 +14,53 @@ explicit name list using Grafana'saction, pinned to that reviewed commit. It reads the triggering attempt's GitHubmetadata and sends OTLP/HTTP protobuf traces to our existing endpoint. The tracehierarchy is workflow → jobs → steps, stored in Tempo with-`service.name=github-actions` and `deployment.environment=ci`. There is no custom-telemetry exporter, triggering PR checkout, artifact download, raw build-log-collection, or Datadog export. Individual test cases are outside this integration.+`service.name=github-actions` and `deployment.environment=ci`. There is no+triggering PR checkout, artifact download, raw build-log collection, or Datadog+export. Individual test cases are outside this integration.++### CI events in Loki++Tempo refuses metrics queries over three hours and keeps traces for seven days, so+it cannot answer daily percentiles. The exporter therefore also writes one log line+per finished push, job and step to Loki, with `service_name="ci-telemetry"`. They+keep for 30 days and give exact percentiles. Fields are structured metadata, read+without a parser; each line's `trace_id` links to the run's trace.++| `event` | Fields |+| --- | --- |+| `push` | `pr_type`, `wall_seconds`, `wall_start`, `start_lag_seconds`, `conclusion`, `last_workflow`, `last_job`, `slowest_job`, `queue_max_seconds`, `cold_volumes`, `files_changed`, `rust_crates_changed`, `experiment`, `rerun`, `pr_draft` |+| `job` | `workflow`, `job`, `conclusion`, `run_seconds`, `queue_seconds`, `runner_profile`, `volume_state`, `dev_shell_seconds` |+| `step` | `workflow`, `job`, `step`, `conclusion`, `run_seconds` |+| `export` | The exporter's own cost: `collect_seconds`, `api_calls`, `rate_remaining`, `rate_limit` |++`pr_type` is `web`, `rust-small`, `rust-medium`, `rust-large`, `mixed`, `infra`,+`other` or `unknown`. Rust size counts changed crates: up to 2, up to 9, then large,+or any root manifest change. `wall_seconds` runs from the push, or from opening the+PR when that came later, to the last job. If neither fits, such as a reopen or+leaving draft, it runs from the first workflow run and `wall_start` says+`first_run`. A push counts as finished when every workflow it started has finished,+so a workflow a PR runs must be on the exporter's list; a test enforces this. A run+that starts after everything on its head has finished came from something else, such+as a PR edit, a label or leaving draft. It forms its own line with `rerun=true`,+which also leaves a commit's first full run after draft out of the real PR numbers.++Real PR numbers exclude experiments, reruns and drafts. Metadata makes an ungrouped+query return one series per combination, so always group:++```logql+quantile_over_time(0.9, {service_name="ci-telemetry"} | event="push"+ | experiment="false" | rerun="false" | pr_draft="false" | conclusion="success"+ | unwrap wall_seconds [1d]) by (pr_type)+```++Lines never carry branch names, actors, titles or file paths. A line's time and+body are the same from every exporter that sees it, so Loki drops the duplicate+when two workflows finish together. A workflow that starts after the rest finished+produces a second `push` line for that head. `volume_state` is `cold` when the dev+shell step takes 30 s or more, an inference from step timing.++Collection and sending are best effort and cannot fail the exporter, and neither+touches the observed PR's jobs. Collection runs before any credential is read.`/d/macro-ci` provides workflow traces, failed/cancelled jobs and steps, and jobsrunning longer than five minutes. Open a trace for its execution waterfall anddiff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rsindex ccbd4f884e..a292ef165e 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs@@ -1,13 +1,21 @@-//! Export completed GitHub Actions attempts as workflow/job/step traces.+//! Export completed GitHub Actions attempts as workflow/job/step traces, and as+//! push, job and step events for percentiles over days.use gh_workflow::{Event, Expression, Job, Level, Permissions, Step, Workflow, WorkflowRun, WorkflowRunType,};-// Explicit names avoid recursively observing this exporter. Add new workflows here.+#[cfg(test)]+mod test;++// Explicit names avoid recursively observing this exporter. Add new workflows here,+// and every workflow a PR runs: a push counts as finished when these have finished.const OBSERVED_WORKFLOWS: &[&str] = &[+ "Android Play Release",+ "Android Release Drift","Auto Label PRs","Build Agent Daemon on Tag",+ "Build CEF with codecs","Build AppImage","Build Desktop on Tag","Build Rust CI Image",@@ -31,15 +39,19 @@ const OBSERVED_WORKFLOWS: &[&str] = &["Ensure Daytona Snapshot","Export Datadog observability definitions","Migrate Macro Database",+ "Native Check Label",+ "PR Privacy Check","Publish Android to Google Play","Publish SDK","Pull Request","Pulumi Preview on PR","Push local stack binaries","Release SDK",+ "Release Label Check","Reusable Deploy Service","Reusable Preview Service","SDK Check",+ "Test macrod update","Web App Pr Checks","cargo deny check","cargo workspace dependency check",@@ -50,10 +62,13 @@ const OBSERVED_WORKFLOWS: &[&str] = &["conventions code check","deploy pulumi service","iOS App Store Publish",+ "iOS App Store Release","iOS Preview",+ "iOS Release Drift","infra code check",];const HAS_JOBS: &str = "steps.preflight.outputs.export == 'true'";+const EVENTS_FILE: &str = "${{ runner.temp }}/ci-telemetry-events.json";/// Build the CI telemetry observer without checking out code or downloading artifacts.pub fn export_ci_telemetry() -> Workflow {@@ -71,15 +86,24 @@ pub fn export_ci_telemetry() -> Workflow {.permissions(Permissions {contents: Some(Level::Read),actions: Some(Level::Read),+ pull_requests: Some(Level::Read),id_token: Some(Level::Write),..Default::default()}).add_job("export", export())}+/// Run embedded TypeScript on the hosted image's Node, which strips types itself, so+/// the exporter needs no checkout, build or dependency.+fn node_script(source: &str) -> String {+ format!(+ "node --disable-warning=ExperimentalWarning --input-type=module-typescript <<'TELEMETRY_SCRIPT'\n{source}\nawait main();\nTELEMETRY_SCRIPT\n"+ )+}+fn export() -> Job {Job::default()- // GitHub's hosted image supplies AWS CLI and Python without a checkout/setup step.+ // GitHub's hosted image supplies AWS CLI, Python and Node without a checkout/setup step..runs_on("ubuntu-latest").timeout_minutes(10u32).add_step(@@ -92,6 +116,17 @@ fn export() -> Job {.id("preflight").add_with(("script", include_str!("scripts/ci_telemetry_preflight.js"))),)+ // Before any credential exists, so nothing the run produced shares a step with it.+ .add_step(+ Step::new("Collect push, job and step events")+ .run(node_script(include_str!("scripts/ci_telemetry_collect.ts")))+ .shell("bash")+ .if_condition(Expression::new("always()"))+ .continue_on_error(true)+ .timeout_minutes(2u32)+ .add_env(("GITHUB_TOKEN", "${{ github.token }}"))+ .add_env(("CI_TELEMETRY_EVENTS", EVENTS_FILE)),+ ).add_step(Step::new("Assume CI telemetry role").uses(@@ -99,7 +134,7 @@ fn export() -> Job {"configure-aws-credentials","e7f100cf4c008499ea8adda475de1042d6975c7b",)- .if_condition(Expression::new(HAS_JOBS))+ .if_condition(Expression::new("always()")).add_with(("role-to-assume","arn:aws:iam::569036502058:role/macro-observability-ci",@@ -111,9 +146,22 @@ fn export() -> Job {Step::new("Read ingestion credential").run(include_str!("scripts/read_ci_ingestion_credential.sh")).id("ingest")- .if_condition(Expression::new(HAS_JOBS))+ .if_condition(Expression::new("always()")).shell("bash"),)+ // Ahead of the traces: one failing must not cost the other, and this one is+ // the only record of a run the trace exporter refuses.+ .add_step(+ Step::new("Send push, job and step events")+ .run(node_script(include_str!("scripts/ci_telemetry_send.ts")))+ .shell("bash")+ .if_condition(Expression::new("always()"))+ .continue_on_error(true)+ .timeout_minutes(1u32)+ .add_env(("CI_TELEMETRY_EVENTS", EVENTS_FILE))+ .add_env(("OTLP_LOGS_URL", "https://otlp.macro-internal.com/v1/logs"))+ .add_env(("OTLP_HEADERS", "${{ steps.ingest.outputs.headers }}")),+ ).add_step(Step::new("Export workflow, job and step traces").uses(diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry/test.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry/test.rsnew file mode 100644index 0000000000..4d81a11a22--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry/test.rs@@ -0,0 +1,104 @@+use super::*;++fn step_names() -> Vec<String> {+ let job = serde_json::to_value(export()).unwrap();+ job["steps"]+ .as_array()+ .unwrap()+ .iter()+ .map(|step| step["name"].as_str().unwrap().to_owned())+ .collect()+}++/// The collector reads data a run produced, so it must run before any credential.+#[test]+fn events_are_collected_before_any_credential_exists() {+ let names = step_names();+ let position = |name: &str| names.iter().position(|step| step == name).unwrap();+ assert!(position("Collect push, job and step events") < position("Assume CI telemetry role"));+ assert!(position("Read ingestion credential") < position("Send push, job and step events"));+}++/// The event steps are best effort, and must run when the trace preflight refuses a run.+#[test]+fn event_steps_cannot_fail_the_exporter_and_survive_preflight() {+ let job = serde_json::to_value(export()).unwrap();+ for step in job["steps"].as_array().unwrap() {+ let name = step["name"].as_str().unwrap();+ if name.ends_with("push, job and step events") {+ assert_eq!(step["continue-on-error"], true, "{name}");+ assert_eq!(step["if"], "always()", "{name}");+ }+ }+}++#[test]+fn only_the_send_step_sees_the_ingestion_headers() {+ let job = serde_json::to_value(export()).unwrap();+ for step in job["steps"].as_array().unwrap() {+ if step["name"] == "Collect push, job and step events" {+ assert!(step["env"]["OTLP_HEADERS"].is_null());+ }+ }+}++/// A push counts as finished once every check has, and only a listed workflow's+/// completion triggers the exporter, so an unlisted one that finishes last loses the push.+#[test]+fn every_pull_request_workflow_is_observed() {+ let workflows = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../../.github/workflows");+ let mut missing = Vec::new();+ for entry in std::fs::read_dir(workflows).unwrap() {+ let path = entry.unwrap().path();+ if !matches!(+ path.extension().and_then(|extension| extension.to_str()),+ Some("yml" | "yaml")+ ) {+ continue;+ }+ let workflow: serde_yaml::Value =+ serde_yaml::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();+ let triggers = &workflow["on"];+ let on_pull_request = ["pull_request", "pull_request_target"].iter().any(|name| {+ triggers.get(*name).is_some()+ || triggers.as_str() == Some(name)+ || triggers+ .as_sequence()+ .is_some_and(|list| list.iter().any(|item| item.as_str() == Some(name)))+ });+ let name = workflow["name"].as_str().unwrap_or_default();+ if on_pull_request && !OBSERVED_WORKFLOWS.contains(&name) {+ missing.push(name.to_owned());+ }+ }+ assert!(missing.is_empty(), "add to OBSERVED_WORKFLOWS: {missing:?}");+}++/// Actions would expand this in the embedded source before the shell ever saw it.+#[test]+fn embedded_scripts_contain_no_actions_expressions() {+ for script in [+ include_str!("../scripts/ci_telemetry_collect.ts"),+ include_str!("../scripts/ci_telemetry_send.ts"),+ ] {+ assert!(!script.contains("${{"));+ }+}++#[test]+fn collector_scenarios() {+ let output = std::process::Command::new("node")+ .args(["--test", "ci_telemetry.test.ts"])+ .current_dir(concat!(+ env!("CARGO_MANIFEST_DIR"),+ "/src/workflows/scripts"+ ))+ .output()+ .expect("run CI telemetry scenarios with Node");+ assert!(+ output.status.success(),+ "CI telemetry scenarios failed:\n{}\n{}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry.test.ts b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry.test.tsnew file mode 100644index 0000000000..aef763bbd3--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry.test.ts@@ -0,0 +1,420 @@+// Scenarios for the CI telemetry collector and sender. Run by xtask_workflows' tests.++import assert from 'node:assert/strict';+import { describe, test } from 'node:test';+import {+ classifyFiles,+ clockStart,+ collect,+ collectPush,+ DEV_SHELL_STEP,+ type GithubApi,+ type Job,+ jobEvents,+ MAX_EVENT_AGE_SECONDS,+ type PullRequest,+ pushEvent,+ type Run,+ sameTriggerRuns,+ type Step,+ toOtlp,+ traceIds,+} from './ci_telemetry_collect.ts';+import { authorization } from './ci_telemetry_send.ts';++const REPO = 'acme/repo';++function at(minute: number, second = 0) {+ const pad = (value: number) => String(value).padStart(2, '0');+ return `2026-01-01T10:${pad(minute)}:${pad(second)}Z`;+}++function when(minute: number, second = 0) {+ return new Date(at(minute, second));+}++function run(overrides: Partial<Run> = {}): Run {+ return {+ id: 1,+ name: 'code check',+ run_attempt: 1,+ event: 'pull_request',+ head_sha: 'abc',+ head_branch: 'someone/some-change',+ status: 'completed',+ conclusion: 'success',+ created_at: at(0, 10),+ updated_at: at(5),+ pull_requests: [{ number: 7 }],+ ...overrides,+ };+}++function job(overrides: Partial<Job> = {}): Job {+ return {+ id: 11,+ name: 'test',+ conclusion: 'success',+ created_at: at(0, 40),+ started_at: at(1),+ completed_at: at(5),+ labels: ['namespace-profile-linux-large;overrides.cache-tag=x'],+ steps: [],+ ...overrides,+ };+}++function step(name: string, start: string, end: string): Step {+ return { name, started_at: start, completed_at: end, conclusion: 'success' };+}++function jobsMap(entries: Record<number, Job[]>) {+ return new Map(Object.entries(entries).map(([id, jobs]) => [Number(id), jobs]));+}++type PushOptions = {+ pushed?: Date | null;+ files?: readonly string[] | null;+ pullRequest?: PullRequest | null;+ head?: Run;+ repeat?: number;+};++function push(runs: Run[], jobs: Record<number, Job[]>, options: PushOptions = {}) {+ const event = pushEvent(+ REPO,+ options.head ?? runs[0],+ runs,+ jobsMap(jobs),+ options.pushed ?? null,+ options.pullRequest ?? null,+ options.files === undefined ? ['crates/a/src/lib.rs'] : options.files,+ options.repeat ?? 0+ );+ assert.ok(event);+ return event;+}++describe('classifyFiles', () => {+ test('types', () => {+ const cases: [string[], string][] = [+ [['apps/web/src/a.tsx'], 'web'],+ [['infra/stacks/a.ts', '.github/workflows/a.yml'], 'infra'],+ [['docs/a.md'], 'other'],+ [['crates/a/src/lib.rs', 'apps/web/b.ts'], 'mixed'],+ [['crates/a/src/lib.rs'], 'rust-small'],+ [['crates/a/src/lib.rs', 'docs/a.md', 'infra/a.ts'], 'rust-small'],+ [Array.from({ length: 5 }, (_, i) => `crates/c${i}/src/lib.rs`), 'rust-medium'],+ [Array.from({ length: 10 }, (_, i) => `crates/c${i}/src/lib.rs`), 'rust-large'],+ [['Cargo.lock'], 'rust-large'],+ ];+ for (const [paths, expected] of cases) {+ assert.equal(classifyFiles(paths)[0], expected, paths.join());+ }+ });++ test('nested crates count separately', () => {+ const paths = Array.from({ length: 4 }, (_, i) => `crates/client/c${i}/src/lib.rs`);+ assert.deepEqual(classifyFiles(paths), ['rust-medium', 4]);+ });++ test('one crate counts once across its directories', () => {+ const paths = ['crates/a/src/lib.rs', 'crates/a/tests/b.rs', 'crates/a/Cargo.toml'];+ assert.deepEqual(classifyFiles(paths), ['rust-small', 1]);+ });++ test('a migration counts as Rust', () => {+ assert.equal(classifyFiles(['crates/db/migrations/1_a.sql'])[0], 'rust-small');+ });++ test('xtask Rust counts as Rust, not infra', () => {+ assert.equal(classifyFiles(['tooling/xtask/src/a.rs'])[0], 'rust-small');+ });+});++describe('clockStart', () => {+ test('the push starts the clock', () => {+ const dayBefore = new Date(when(0).getTime() - 86_400_000);+ assert.deepEqual(clockStart(when(0, 10), when(0), dayBefore), [when(0), 'push']);+ });++ test('opening the PR after the push starts the clock', () => {+ assert.deepEqual(clockStart(when(5, 10), when(0), when(5)), [when(5), 'pr_open']);+ });++ test('an old push falls back to the first run', () => {+ assert.deepEqual(clockStart(when(30), when(0), null), [when(30), 'first_run']);+ });++ test('an unknown push falls back to the first run', () => {+ assert.deepEqual(clockStart(when(0), null, when(0)), [when(0), 'first_run']);+ });+});++describe('jobEvents', () => {+ test('a job has queue and volume', () => {+ const steps = [step(DEV_SHELL_STEP, at(1), at(1, 4)), step('instant', at(1), at(1))];+ const events = jobEvents(REPO, run(), [job({ steps })]);+ const { attributes } = events[0];+ assert.equal(attributes.queue_seconds, 20);+ assert.equal(attributes.run_seconds, 240);+ assert.equal(attributes.volume_state, 'warm');+ assert.equal(attributes.runner_profile, 'namespace-profile-linux-large');+ assert.deepEqual(+ events.map((event) => event.attributes.event),+ ['job', 'step']+ );+ });++ test('a slow dev shell means cold', () => {+ const steps = [step(DEV_SHELL_STEP, at(1), at(3))];+ assert.equal(jobEvents(REPO, run(), [job({ steps })])[0].attributes.volume_state, 'cold');+ });++ test("a partial rerun's copy has no queue time", () => {+ const { attributes } = jobEvents(REPO, run(), [job({ created_at: at(9) })])[0];+ assert.ok(!('queue_seconds' in attributes));+ });++ test('a skipped job is left out', () => {+ assert.deepEqual(jobEvents(REPO, run(), [job({ conclusion: 'skipped' })]), []);+ });++ test('trace ids match the exporter', () => {+ // A real run and the id its trace has in Tempo.+ const [traceId, spanId] = traceIds('macro-inc/macro', { id: 42073661884, run_attempt: 1 });+ assert.equal(traceId, '6957c75506a4f45cba526240dd419983');+ assert.equal(spanId.length, 16);+ });+});++describe('pushEvent', () => {+ test('wall is push to last job across workflows', () => {+ const runs = [run(), run({ id: 2, name: 'web' })];+ const jobs = { 1: [job({ completed_at: at(5) })], 2: [job({ name: 'build', completed_at: at(7), id: 12 })] };+ const { at: end, attributes } = push(runs, jobs, { pushed: when(0) });+ assert.equal(attributes.wall_seconds, 420);+ assert.equal(attributes.wall_start, 'push');+ assert.equal(attributes.start_lag_seconds, 10);+ assert.equal(attributes.last_job, 'build');+ assert.equal(attributes.last_workflow, 'web');+ assert.deepEqual(end, when(7));+ });++ test('a roll-up job ends the wall but is not the last job', () => {+ const jobs = {+ 1: [+ job({ completed_at: at(5) }),+ job({ name: 'Cloud Storage Status Check', started_at: at(5), completed_at: at(6), id: 12 }),+ ],+ };+ const { attributes } = push([run()], jobs);+ assert.equal(attributes.last_job, 'test');+ assert.equal(attributes.wall_seconds, 350);+ });++ test('the same push gives the same line from either completion', () => {+ const runs = [run(), run({ id: 2, name: 'web' })];+ const jobs = { 1: [job()], 2: [job({ name: 'build', id: 12 })] };+ const first = push(runs, jobs);+ const second = push(runs, jobs, { head: runs[1] });+ assert.equal(first.body, second.body);+ assert.deepEqual(first.at, second.at);+ });++ test('a rerun is flagged and gets a new line', () => {+ const first = push([run()], { 1: [job()] });+ const rerun = push([run({ run_attempt: 2 })], { 1: [job()] });+ assert.equal(rerun.attributes.rerun, true);+ assert.notEqual(first.body, rerun.body);+ });++ test('any failure fails the push', () => {+ const runs = [run(), run({ id: 2, name: 'web', conclusion: 'failure' })];+ const { attributes } = push(runs, { 1: [job()], 2: [job({ id: 12 })] });+ assert.equal(attributes.conclusion, 'failure');+ });++ test("a workflow's newest run decides the conclusion", () => {+ const runs = [run({ conclusion: 'failure' }), run({ id: 2, created_at: at(2) })];+ const { attributes } = push(runs, { 1: [job()], 2: [job({ id: 12 })] });+ assert.equal(attributes.conclusion, 'success');+ });++ test('a later trigger on the same head is flagged', () => {+ const event = push([run()], { 1: [job()] }, { repeat: 1 });+ assert.equal(event.attributes.rerun, true);+ assert.notEqual(event.body, push([run()], { 1: [job()] }).body);+ });++ test('no job ran is no event', () => {+ assert.equal(pushEvent(REPO, run(), [run()], jobsMap({ 1: [] }), null, null, []), null);+ });++ test('an experiment branch is flagged and never named', () => {+ const head = run({ head_branch: 'someone/ci-probe' });+ const { attributes } = push([head], { 1: [job()] });+ assert.equal(attributes.experiment, true);+ assert.ok(!JSON.stringify(attributes).includes('someone'));+ });++ test('no PR is unknown type', () => {+ assert.equal(push([run()], { 1: [job()] }, { files: null }).attributes.pr_type, 'unknown');+ });+});++describe('sameTriggerRuns', () => {+ test('overlapping runs are one group', () => {+ const runs = [run(), run({ id: 2, created_at: at(4, 11), updated_at: at(9) })];+ assert.deepEqual(sameTriggerRuns(runs, runs[1]), [runs, 0]);+ });++ test('a run joining late extends the group', () => {+ const runs = [+ run(),+ run({ id: 2, created_at: at(4), updated_at: at(9) }),+ run({ id: 3, created_at: at(8), updated_at: at(10) }),+ ];+ assert.deepEqual(sameTriggerRuns(runs, runs[2]), [runs, 0]);+ });++ test('an edit after everything finished is its own group', () => {+ const first = run({ updated_at: at(1) });+ const later = run({ id: 2, created_at: at(2), updated_at: at(3) });+ assert.deepEqual(sameTriggerRuns([later, first], first), [[first], 0]);+ assert.deepEqual(sameTriggerRuns([later, first], later), [[later], 1]);+ });++ test('a run still going keeps the group open', () => {+ const first = run({ status: 'in_progress' });+ const later = run({ id: 2, created_at: at(40) });+ assert.deepEqual(sameTriggerRuns([first, later], later), [[first, later], 0]);+ });++ test('a run outside the checks has no group', () => {+ assert.deepEqual(sameTriggerRuns([run()], run({ id: 9 })), [[], 0]);+ });+});++class FakeGithub implements GithubApi {+ responses: Record<string, unknown>;+ calls = 0;+ rateLimit = null;+ rateRemaining = null;++ constructor(responses: Record<string, unknown>) {+ this.responses = responses;+ }++ async get(path: string) {+ this.calls += 1;+ const response = this.responses[path];+ if (response instanceof Error) throw response;+ return response;+ }+}++function responses(...runs: Run[]): Record<string, unknown> {+ return {+ 'actions/runs': { workflow_runs: runs },+ 'actions/runs/1/attempts/1/jobs': { jobs: [job()] },+ 'actions/runs/2/attempts/1/jobs': { jobs: [job({ name: 'build', id: 12 })] },+ 'pulls/7': { created_at: at(0), draft: false },+ 'pulls/7/files': [{ filename: 'apps/web/a.ts' }],+ activity: [{ after: 'abc', timestamp: at(0, 5) }],+ };+}++describe('collectPush', () => {+ test('waits for every check', async () => {+ const github = new FakeGithub(+ responses(run(), run({ id: 2, status: 'in_progress', conclusion: null }))+ );+ assert.equal(await collectPush(github, REPO, run()), null);+ assert.equal(github.calls, 1);+ });++ test('a finished push is classified', async () => {+ const github = new FakeGithub(responses(run(), run({ id: 2 })));+ const event = await collectPush(github, REPO, run());+ assert.equal(event?.attributes.pr_type, 'web');+ assert.equal(event?.attributes.wall_start, 'push');+ });++ test('a run without its PR still finds the PR through its group', async () => {+ const last = run({ id: 2, pull_requests: [] });+ const github = new FakeGithub(responses(run(), last));+ const event = await collectPush(github, REPO, last);+ assert.equal(event?.attributes.pr_type, 'web');+ assert.equal(event?.attributes.pr_number, 7);+ });++ test('a later edit does not hold up the push', async () => {+ const github = new FakeGithub(+ responses(run(), run({ id: 2, created_at: at(40), status: 'in_progress' }))+ );+ const event = await collectPush(github, REPO, run());+ assert.equal(event?.attributes.rerun, false);+ });++ test('cleanup and other events do not block', async () => {+ const github = new FakeGithub(+ responses(+ run(),+ run({ id: 3, name: 'Cleanup Preview', status: 'in_progress' }),+ run({ id: 4, event: 'push', status: 'in_progress' })+ )+ );+ assert.notEqual(await collectPush(github, REPO, run()), null);+ });++ test('a main push has no push event', async () => {+ const github = new FakeGithub({});+ assert.equal(await collectPush(github, REPO, run({ event: 'push' })), null);+ assert.equal(github.calls, 0);+ });++ test('an error keeps the job events', async () => {+ const github = new FakeGithub({ ...responses(run()), 'actions/runs': new Error('boom') });+ const events = await collect(github, REPO, run());+ assert.deepEqual(+ events.map((event) => event.attributes.event),+ ['job']+ );+ });+});++describe('toOtlp', () => {+ test('records are typed and clamped', () => {+ const now = when(59);+ const document = toOtlp(+ [+ {+ body: 'line',+ at: when(0),+ attributes: { a: 1, b: 'x', c: true },+ ids: ['t'.repeat(32), 's'.repeat(16)],+ },+ ],+ now+ );+ const record = document.resourceLogs[0].scopeLogs[0].logRecords[0];+ const oldest = Math.trunc(now.getTime() / 1000) - MAX_EVENT_AGE_SECONDS;+ assert.equal(record.timeUnixNano, `${oldest}000000000`);+ assert.deepEqual(record.attributes, [+ { key: 'a', value: { intValue: '1' } },+ { key: 'b', value: { stringValue: 'x' } },+ { key: 'c', value: { boolValue: true } },+ ]);+ assert.equal(record.traceId, 't'.repeat(32));+ });+});++describe('authorization', () => {+ test("reads the credential step's header form", () => {+ assert.deepEqual(authorization('Authorization=Bearer%20a%2Fb'), [+ 'Authorization',+ 'Bearer a/b',+ ]);+ });+});diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_collect.ts b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_collect.tsnew file mode 100644index 0000000000..8a4d3b691e--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_collect.ts@@ -0,0 +1,591 @@+// Turn one completed workflow run into CI telemetry events, as an OTLP logs file.+//+// Runs in the privileged exporter without the ingestion token. Everything here is+// best effort: an error becomes a warning, and the observed workflow is never+// affected. Events carry ids, names of our own jobs and steps, and timings, never+// branch names, actors, titles or file paths.++import { createHash } from 'node:crypto';+import { readFileSync, writeFileSync } from 'node:fs';++const API = 'https://api.github.com';+const PR_EVENTS = new Set(['pull_request', 'pull_request_target']);+// Runs after a PR closes, so it is never part of waiting for checks.+const NOT_A_CHECK = new Set(['Cleanup Preview']);+// Experiment branches stay queryable but are flagged out of the real PR numbers.+const EXPERIMENT_BRANCH = /^[^/]+\/ci-/;+// A warm volume restores the dev shell in about 4 s, an empty one takes 110 s or more.+export const DEV_SHELL_STEP = 'Setup Nix dev shell';+const COLD_VOLUME_SECONDS = 30;+// A start this long after the push was not caused by it: a reopen or leaving draft.+const MAX_PUSH_LAG_SECONDS = 600;+// Below this a step is noise, and it stays visible in the trace.+const MIN_STEP_SECONDS = 1;+// Stop before the exporter's own traces start failing for lack of API budget.+const MIN_RATE_REMAINING = 100;+const MAX_FILE_PAGES = 3;+// Roll-up jobs finish last by design, so they never say what held a push up.+const ROLL_UP_JOB = /status[ -]check/i;+// Loki refuses a line this far behind its stream's newest one.+export const MAX_EVENT_AGE_SECONDS = 1800;+const SMALL_RUST_CRATES = 2;+const MEDIUM_RUST_CRATES = 9;+const WEB_SUFFIXES = ['.ts', '.tsx', '.js', '.jsx', '.css', '.html', '.svelte'];+const RUST_ROOT_FILES = new Set(['Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml']);+const CRATE_DIRECTORIES = new Set(['src', 'tests', 'benches', 'examples', 'migrations']);++export type Timed = {+ started_at?: string | null;+ completed_at?: string | null;+ conclusion?: string | null;+};++export type Step = Timed & { name: string };++export type Job = Timed & {+ id: number;+ name: string;+ created_at?: string | null;+ labels?: string[] | null;+ steps?: Step[] | null;+};++export type Run = {+ id: number;+ name: string;+ run_attempt: number;+ event: string;+ head_sha: string;+ head_branch?: string | null;+ status: string;+ conclusion: string | null;+ created_at: string;+ updated_at: string;+ pull_requests?: { number: number }[] | null;+ head_repository?: { fork?: boolean } | null;+};++export type PullRequest = { created_at: string; draft?: boolean };++type Interval = [Date, Date];+type Attributes = Record<string, string | number | boolean>;++export type TelemetryEvent = {+ body: string;+ at: Date;+ attributes: Attributes;+ ids: [traceId: string, spanId: string];+};++export interface GithubApi {+ get(path: string, params?: Record<string, string | number>): Promise<any>;+ calls: number;+ rateLimit: number | null;+ rateRemaining: number | null;+}++function parseTime(value: string | null | undefined): Date | null {+ return value ? new Date(value) : null;+}++function seconds(start: Date, end: Date): number {+ return Math.trunc((end.getTime() - start.getTime()) / 1000);+}++function maxBy<T>(items: T[], key: (item: T) => number): T {+ return items.reduce((best, item) => (key(item) > key(best) ? item : best));+}++export class HttpError extends Error {+ code: number;++ constructor(code: number) {+ super(`HTTP ${code}`);+ this.code = code;+ }+}++/** Counts its own calls and stops before exhausting the shared token budget. */+class Github implements GithubApi {+ repo: string;+ token: string;+ calls = 0;+ rateLimit: number | null = null;+ rateRemaining: number | null = null;++ constructor(repo: string, token: string) {+ this.repo = repo;+ this.token = token;+ }++ async get(path: string, params: Record<string, string | number> = {}) {+ if (this.rateRemaining !== null && this.rateRemaining < MIN_RATE_REMAINING) {+ throw new Error('GitHub API budget is low');+ }+ const query = new URLSearchParams(+ Object.entries(params).map(([key, value]): [string, string] => [key, String(value)])+ ).toString();+ this.calls += 1;+ const response = await fetch(`${API}/repos/${this.repo}/${path}${query ? `?${query}` : ''}`, {+ headers: {+ Authorization: `Bearer ${this.token}`,+ Accept: 'application/vnd.github+json',+ },+ signal: AbortSignal.timeout(20_000),+ });+ if (!response.ok) throw new HttpError(response.status);+ this.rateLimit = Number(response.headers.get('x-ratelimit-limit')) || null;+ this.rateRemaining = Number(response.headers.get('x-ratelimit-remaining')) || 0;+ return response.json();+ }+}++/** The directory holding a crate's sources, which may be nested in a workspace. */+function crateRoot(parts: string[]): string {+ for (const [index, part] of parts.slice(0, -1).entries()) {+ if (CRATE_DIRECTORIES.has(part)) return parts.slice(0, index).join('/');+ }+ return parts.slice(0, -1).join('/') || parts[0];+}++/**+ * Return [pr_type, rust_crates_changed] from the changed paths.+ *+ * Sizes count changed crates, because the workflow API can't see how many packages+ * path-check selected.+ */+export function classifyFiles(paths: readonly string[]): [string, number] {+ const crates = new Set<string>();+ let rootManifest = false;+ let web = false;+ let infra = false;+ for (const path of paths) {+ const parts = path.split('/');+ if (parts[0] === 'infra' || parts[0] === '.github' || path.endsWith('.nix')) {+ infra = true;+ } else if (+ path.endsWith('.rs') ||+ parts[0] === 'crates' ||+ parts[0] === '.sqlx' ||+ RUST_ROOT_FILES.has(parts[parts.length - 1])+ ) {+ if (parts.length === 1) rootManifest = true;+ crates.add(crateRoot(parts));+ } else if (+ parts[0] === 'apps' ||+ parts[0] === 'packages' ||+ WEB_SUFFIXES.some((suffix) => path.endsWith(suffix))+ ) {+ web = true;+ } else if (parts[0] === 'tooling') {+ infra = true;+ }+ }+ if (crates.size > 0 && web) return ['mixed', crates.size];+ if (crates.size > 0) {+ if (rootManifest || crates.size > MEDIUM_RUST_CRATES) return ['rust-large', crates.size];+ return [crates.size <= SMALL_RUST_CRATES ? 'rust-small' : 'rust-medium', crates.size];+ }+ if (web) return ['web', 0];+ return [infra ? 'infra' : 'other', 0];+}++function runAttributes(run: Run): Attributes {+ const attributes: Attributes = {+ workflow: run.name,+ run_id: run.id,+ run_attempt: run.run_attempt,+ event_name: run.event,+ head_sha: run.head_sha,+ experiment: EXPERIMENT_BRANCH.test(run.head_branch ?? ''),+ rerun: run.run_attempt > 1,+ };+ const pullRequest = run.pull_requests?.[0];+ if (pullRequest) attributes.pr_number = pullRequest.number;+ return attributes;+}++/** Reproduce gha-otel-export's ids, so an event links to its workflow trace. */+export function traceIds(+ repo: string,+ run: Pick<Run, 'id' | 'run_attempt'>,+ suffix?: string+): [string, string] {+ const prefix = `${repo}${run.id}${run.run_attempt}`;+ const hash = (input: string) => createHash('sha256').update(input).digest('hex');+ return [hash(`${prefix}t`).slice(0, 32), hash(`${prefix}${suffix ?? 's'}`).slice(16, 32)];+}++/** Started and finished, not skipped, and with times GitHub didn't copy from an old attempt. */+function timed(entry: Timed): Interval | null {+ const start = parseTime(entry.started_at);+ const end = parseTime(entry.completed_at);+ if (entry.conclusion === 'skipped' || !start || !end || end < start) return null;+ return [start, end];+}++function volumeState(job: Job): [string, number] | null {+ for (const step of job.steps ?? []) {+ const times = timed(step);+ if (step.name === DEV_SHELL_STEP && times) {+ const duration = seconds(...times);+ return [duration >= COLD_VOLUME_SECONDS ? 'cold' : 'warm', duration];+ }+ }+ return null;+}++/** Null for a partial rerun's copied job, which starts before it was created. */+function queueSeconds(job: Job, times: Interval): number | null {+ const created = parseTime(job.created_at);+ if (!created || times[0] < created) return null;+ return seconds(created, times[0]);+}++/** One event per job that ran, and one per step long enough to matter. */+export function jobEvents(repo: string, run: Run, jobs: Job[]): TelemetryEvent[] {+ const base = runAttributes(run);+ const events: TelemetryEvent[] = [];+ for (const job of jobs) {+ const times = timed(job);+ if (!times) continue;+ const attributes: Attributes = {+ ...base,+ event: 'job',+ job: job.name,+ job_id: job.id,+ conclusion: job.conclusion ?? '',+ run_seconds: seconds(...times),+ };+ const queue = queueSeconds(job, times);+ if (queue !== null) attributes.queue_seconds = queue;+ const label = job.labels?.[0];+ if (label) attributes.runner_profile = label.split(';')[0];+ const volume = volumeState(job);+ if (volume) {+ attributes.volume_state = volume[0];+ attributes.dev_shell_seconds = volume[1];+ }+ events.push({+ body: `job ${job.id} ${run.run_attempt}`,+ at: times[1],+ attributes,+ ids: traceIds(repo, run, job.name),+ });+ for (const [number, step] of (job.steps ?? []).entries()) {+ const stepTimes = timed(step);+ if (!stepTimes || seconds(...stepTimes) < MIN_STEP_SECONDS) continue;+ events.push({+ body: `step ${job.id} ${number} ${run.run_attempt}`,+ at: stepTimes[1],+ attributes: {+ ...base,+ event: 'step',+ job: job.name,+ job_id: job.id,+ step: step.name,+ conclusion: step.conclusion ?? '',+ run_seconds: seconds(...stepTimes),+ },+ ids: traceIds(repo, run, job.name + step.name),+ });+ }+ }+ return events;+}++/**+ * The runs started together with this one, and how many such groups came before.+ *+ * A run that starts after everything on the head has finished wasn't part of that+ * wait: it came from a PR edit, a label or leaving draft.+ */+export function sameTriggerRuns(runs: Run[], run: Pick<Run, 'id'>): [Run[], number] {+ const created = (entry: Run) => new Date(entry.created_at).getTime();+ const groups: Run[][] = [];+ let groupEnd = 0;+ for (const candidate of [...runs].sort((a, b) => created(a) - created(b))) {+ const end =+ candidate.status === 'completed' ? new Date(candidate.updated_at).getTime() : Infinity;+ const group = groups[groups.length - 1];+ if (group && created(candidate) <= groupEnd) {+ group.push(candidate);+ groupEnd = Math.max(groupEnd, end);+ } else {+ groups.push([candidate]);+ groupEnd = end;+ }+ }+ for (const [index, group] of groups.entries()) {+ if (group.some((candidate) => candidate.id === run.id)) return [group, index];+ }+ return [[], 0];+}++/** A workflow's newest run decides, since a PR edit can correct an earlier failure. */+function conclusionOf(runs: Run[]): string {+ const ordered = [...runs].sort(+ (a, b) => new Date(a.created_at).getTime() - new Date(b.created_at).getTime() || a.id - b.id+ );+ const newest = new Map(ordered.map((run) => [run.name, run]));+ const conclusions = new Set([...newest.values()].map((run) => run.conclusion));+ if (['failure', 'timed_out', 'startup_failure'].some((failed) => conclusions.has(failed))) {+ return 'failure';+ }+ return conclusions.has('cancelled') ? 'cancelled' : 'success';+}++/** When the wait began: the push, or opening the PR when that came later. */+export function clockStart(+ firstCreated: Date,+ pushedAt: Date | null,+ prCreatedAt: Date | null+): [Date, string] {+ if (!pushedAt) return [firstCreated, 'first_run'];+ const start = prCreatedAt && prCreatedAt > pushedAt ? prCreatedAt : pushedAt;+ if (start > firstCreated || seconds(start, firstCreated) > MAX_PUSH_LAG_SECONDS) {+ return [firstCreated, 'first_run'];+ }+ return [start, start === pushedAt ? 'push' : 'pr_open'];+}++type Finished = { end: Date; times: Interval; job: Job; run: Run };++/**+ * The whole push: start to last job, by PR type. Null when no job ran.+ *+ * `repeat` counts earlier groups of runs on this head. Those later groups are not+ * the push's wait, so they are flagged as reruns.+ */+export function pushEvent(+ repo: string,+ run: Run,+ runs: Run[],+ jobsByRun: Map<number, Job[]>,+ pushedAt: Date | null,+ pullRequest: PullRequest | null,+ files: readonly string[] | null,+ repeat = 0+): TelemetryEvent | null {+ const finished: Finished[] = [];+ for (const candidate of runs) {+ for (const job of jobsByRun.get(candidate.id) ?? []) {+ const times = timed(job);+ if (times) finished.push({ end: times[1], times, job, run: candidate });+ }+ }+ if (finished.length === 0) return null;+ const firstCreated = new Date(Math.min(...runs.map((c) => new Date(c.created_at).getTime())));+ const prCreatedAt = pullRequest ? new Date(pullRequest.created_at) : null;+ const [start, startSource] = clockStart(firstCreated, pushedAt, prCreatedAt);+ const end = maxBy(finished, (entry) => entry.end.getTime()).end;+ const notRollUp = finished.filter((entry) => !ROLL_UP_JOB.test(entry.job.name));+ const work = notRollUp.length > 0 ? notRollUp : finished;+ const last = maxBy(work, (entry) => entry.end.getTime());+ const slowest = maxBy(work, (entry) => seconds(...entry.times));+ const queues = finished+ .map((entry) => queueSeconds(entry.job, entry.times))+ .filter((queue) => queue !== null);+ const [prType, rustCrates] = files !== null ? classifyFiles(files) : ['unknown', 0];+ const attributes: Attributes = {+ ...runAttributes(run),+ event: 'push',+ pr_type: prType,+ rust_crates_changed: rustCrates,+ conclusion: conclusionOf(runs),+ wall_seconds: seconds(start, end),+ wall_start: startSource,+ start_lag_seconds: seconds(start, firstCreated),+ last_workflow: last.run.name,+ last_job: last.job.name,+ last_job_seconds: seconds(...last.times),+ slowest_job: slowest.job.name,+ slowest_job_seconds: seconds(...slowest.times),+ queue_max_seconds: Math.max(0, ...queues),+ queue_total_seconds: queues.reduce((sum, queue) => sum + queue, 0),+ workflows: runs.length,+ jobs: finished.length,+ cold_volumes: finished.filter((entry) => volumeState(entry.job)?.[0] === 'cold').length,+ rerun: repeat > 0 || runs.some((candidate) => candidate.run_attempt > 1),+ };+ delete attributes.workflow;+ delete attributes.run_id;+ delete attributes.run_attempt;+ if (files !== null) attributes.files_changed = files.length;+ if (pullRequest) attributes.pr_draft = Boolean(pullRequest.draft);+ if (run.head_repository?.fork) attributes.fork = true;+ const attempts = runs.reduce((sum, candidate) => sum + candidate.run_attempt, 0);+ // The body and time are the same from every exporter that sees the push finish,+ // so Loki drops the duplicates when two runs complete together.+ return {+ body: `push ${run.head_sha} ${repeat} ${attempts}`,+ at: end,+ attributes,+ ids: traceIds(repo, last.run, last.job.name),+ };+}++async function pushedTime(github: GithubApi, branch: string, headSha: string) {+ const activity: { after?: string; timestamp: string }[] = await github.get('activity', {+ ref: `refs/heads/${branch}`,+ per_page: 30,+ });+ const entry = activity.find((candidate) => candidate.after === headSha);+ return entry ? new Date(entry.timestamp) : null;+}++async function changedFiles(github: GithubApi, number: number) {+ const paths: string[] = [];+ for (let page = 1; page <= MAX_FILE_PAGES; page += 1) {+ const batch: { filename: string }[] = await github.get(`pulls/${number}/files`, {+ per_page: 100,+ page,+ });+ paths.push(...batch.map((entry) => entry.filename));+ if (batch.length < 100) break;+ }+ return paths;+}++/** The push event once this completion is the last one, otherwise null. */+export async function collectPush(+ github: GithubApi,+ repo: string,+ run: Run+): Promise<TelemetryEvent | null> {+ if (!PR_EVENTS.has(run.event)) return null;+ const listing: { workflow_runs: Run[] } = await github.get('actions/runs', {+ head_sha: run.head_sha,+ per_page: 100,+ });+ const checks = listing.workflow_runs.filter(+ (candidate) => PR_EVENTS.has(candidate.event) && !NOT_A_CHECK.has(candidate.name)+ );+ const [runs, repeat] = sameTriggerRuns(checks, run);+ if (runs.length === 0 || runs.some((candidate) => candidate.status !== 'completed')) {+ return null;+ }+ const jobsByRun = new Map<number, Job[]>();+ for (const candidate of runs) {+ jobsByRun.set(+ candidate.id,+ candidate.conclusion === 'skipped'+ ? []+ : (+ await github.get(+ `actions/runs/${candidate.id}/attempts/${candidate.run_attempt}/jobs`,+ { per_page: 100 }+ )+ ).jobs+ );+ }+ // GitHub leaves `pull_requests` empty on some runs, so ask the group when this is one.+ const head = run.pull_requests?.length+ ? run+ : (runs.find((candidate) => candidate.pull_requests?.length) ?? run);+ const number = head.pull_requests?.[0]?.number;+ const pullRequest: PullRequest | null = number ? await github.get(`pulls/${number}`) : null;+ const files = number ? await changedFiles(github, number) : null;+ const pushedAt =+ !run.head_repository?.fork && run.head_branch+ ? await pushedTime(github, run.head_branch, run.head_sha)+ : null;+ return pushEvent(repo, head, runs, jobsByRun, pushedAt, pullRequest, files, repeat);+}++function otlpValue(value: string | number | boolean) {+ if (typeof value === 'boolean') return { boolValue: value };+ if (typeof value === 'number') return { intValue: String(value) };+ return { stringValue: value };+}++/** OTLP logs. A record's time is its event's, clamped to what Loki will accept. */+export function toOtlp(events: TelemetryEvent[], now: Date) {+ const oldest = now.getTime() - MAX_EVENT_AGE_SECONDS * 1000;+ const logRecords = events.map(({ body, at, attributes, ids: [traceId, spanId] }) => ({+ timeUnixNano: `${Math.trunc(Math.max(at.getTime(), oldest) / 1000)}000000000`,+ severityText: 'INFO',+ body: { stringValue: body },+ traceId,+ spanId,+ attributes: Object.entries(attributes).map(([key, value]) => ({+ key,+ value: otlpValue(value),+ })),+ }));+ const attributes = [+ { key: 'service.name', value: { stringValue: 'ci-telemetry' } },+ { key: 'deployment.environment', value: { stringValue: 'ci' } },+ ];+ return {+ resourceLogs: [+ {+ resource: { attributes },+ scopeLogs: [{ scope: { name: 'ci-telemetry' }, logRecords }],+ },+ ],+ };+}++export async function collect(github: GithubApi, repo: string, run: Run) {+ const events: TelemetryEvent[] = [];+ try {+ const { jobs } = await github.get(+ `actions/runs/${run.id}/attempts/${run.run_attempt}/jobs`,+ { per_page: 100 }+ );+ events.push(...jobEvents(repo, run, jobs));+ const push = await collectPush(github, repo, run);+ if (push) events.push(push);+ } catch (error) {+ // A partial export still helps. The class and status can't carry a token or a payload.+ console.log(`::warning::CI telemetry partly collected: ${describe(error)}`);+ }+ return events;+}++function describe(error: unknown): string {+ if (error instanceof HttpError) return `HttpError ${error.code}`;+ return error instanceof Error ? error.constructor.name : 'unknown error';+}++function required(name: string): string {+ const value = process.env[name];+ if (!value) throw new Error('A required environment variable is missing');+ return value;+}++async function runCollection() {+ const started = performance.now();+ const repo = required('GITHUB_REPOSITORY');+ const run: Run = JSON.parse(readFileSync(required('GITHUB_EVENT_PATH'), 'utf8')).workflow_run;+ const github = new Github(repo, required('GITHUB_TOKEN'));+ const events = await collect(github, repo, run);+ const now = new Date();+ const counts: Attributes = {+ event: 'export',+ ...runAttributes(run),+ events: events.length,+ api_calls: github.calls,+ collect_seconds: Math.round((performance.now() - started) / 1000),+ };+ if (github.rateRemaining !== null) counts.rate_remaining = github.rateRemaining;+ if (github.rateLimit) counts.rate_limit = github.rateLimit;+ events.push({+ body: `export ${run.id} ${run.run_attempt}`,+ at: now,+ attributes: counts,+ ids: traceIds(repo, run),+ });+ writeFileSync(required('CI_TELEMETRY_EVENTS'), JSON.stringify(toOtlp(events, now)));+ console.log(`${events.length} events, ${github.calls} API calls`);+}++/** Entry point, called by the workflow step and never by an import. */+export async function main() {+ try {+ await runCollection();+ } catch (error) {+ console.log(`::warning::CI telemetry not collected: ${describe(error)}`);+ }+}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_send.ts b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_send.tsnew file mode 100644index 0000000000..1c2eb03faf--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_send.ts@@ -0,0 +1,72 @@+// Send the collected CI telemetry to the OTLP logs route. Best effort, never fails the job.++import { existsSync, readFileSync } from 'node:fs';+import { gzipSync } from 'node:zlib';++const ATTEMPTS = 2;+const TIMEOUT_MILLISECONDS = 10_000;+const RETRY_DELAY_MILLISECONDS = 2_000;++/** Read the `Authorization=Bearer%20...` form the credential step writes. */+export function authorization(headers: string): [string, string] {+ const separator = headers.indexOf('=');+ return [headers.slice(0, separator), decodeURIComponent(headers.slice(separator + 1))];+}++async function send(url: string, body: Buffer, header: [string, string]) {+ const response = await fetch(url, {+ method: 'POST',+ body: gzipSync(body),+ headers: {+ 'Content-Type': 'application/json',+ 'Content-Encoding': 'gzip',+ [header[0]]: header[1],+ },+ signal: AbortSignal.timeout(TIMEOUT_MILLISECONDS),+ });+ if (!response.ok) throw new Error(`HTTP ${response.status}`);+ return response.status;+}++function required(name: string): string {+ const value = process.env[name];+ if (!value) throw new Error('A required environment variable is missing');+ return value;+}++async function sendEvents() {+ const started = performance.now();+ const path = required('CI_TELEMETRY_EVENTS');+ if (!existsSync(path)) {+ console.log('Nothing to send');+ return;+ }+ const body = readFileSync(path);+ const header = authorization(required('OTLP_HEADERS'));+ const url = required('OTLP_LOGS_URL');+ for (let attempt = 1; attempt <= ATTEMPTS; attempt += 1) {+ try {+ const status = await send(url, body, header);+ const elapsed = ((performance.now() - started) / 1000).toFixed(1);+ console.log(`Sent ${body.length} bytes, HTTP ${status}, ${elapsed} s`);+ return;+ } catch (error) {+ // Only the class, so neither the header nor a response reaches the log.+ const reason = error instanceof Error ? error.constructor.name : 'unknown error';+ console.log(`::warning::CI telemetry send attempt ${attempt} failed: ${reason}`);+ if (attempt < ATTEMPTS) {+ await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MILLISECONDS));+ }+ }+ }+}++/** Entry point, called by the workflow step and never by an import. */+export async function main() {+ try {+ await sendEvents();+ } catch (error) {+ const reason = error instanceof Error ? error.constructor.name : 'unknown error';+ console.log(`::warning::CI telemetry not sent: ${reason}`);+ }+}