Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

chore(observability): export CI traces with Grafana action (#7883)

Wolf Mermelstein275ae30486fd
diff --git a/.github/workflows/export-ci-telemetry.yml b/.github/workflows/export-ci-telemetry.yml
new file mode 100644
index 0000000000..8aae61e606
--- /dev/null
+++ b/.github/workflows/export-ci-telemetry.yml
@@ -0,0 +1,138 @@
+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).
+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs
+name: Export CI telemetry
+'on':
+ workflow_run:
+ types:
+ - completed
+ workflows:
+ - Auto Label PRs
+ - Build Agent Daemon on Tag
+ - Build AppImage
+ - Build Desktop on Tag
+ - Build Rust CI Image
+ - Build Web App
+ - Build macOS DMG
+ - CLA
+ - Cancel Stuck Cloud Storage Deploys
+ - Cleanup Preview
+ - Deploy AI Editing Worker
+ - Deploy All Services
+ - Deploy CLA Worker
+ - Deploy FusionAuth Instance
+ - Deploy Lexical Service
+ - Deploy Preview
+ - Deploy Production Release
+ - Deploy Service (Generic)
+ - Deploy Sync Service
+ - Deploy Web App
+ - Deploy Website
+ - Deploy analytics proxy dev
+ - Deploy on Push
+ - Ensure Daytona Snapshot
+ - Export Datadog observability definitions
+ - Migrate Macro Database
+ - Publish Android to Google Play
+ - Publish SDK
+ - Pull Request
+ - Pulumi Preview on PR
+ - Push local stack binaries
+ - Release SDK
+ - Reusable Deploy Service
+ - Reusable Preview Service
+ - SDK Check
+ - Web App Pr Checks
+ - cargo deny check
+ - cargo workspace dependency check
+ - check generated workflows
+ - check node_modules nix
+ - cloud storage code check
+ - conventions code check
+ - deploy pulumi service
+ - iOS Preview
+ - infra code check
+permissions:
+ actions: read
+ contents: read
+ id-token: write
+jobs:
+ export:
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ steps:
+ - id: preflight
+ name: Check upstream exporter limits
+ uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b
+ with:
+ script: |
+ const run = context.payload.workflow_run;
+ const { data } = await github.rest.actions.listJobsForWorkflowRunAttempt({
+ ...context.repo, run_id: run.id, attempt_number: run.run_attempt, per_page: 100,
+ });
+ if (data.total_count > 100 || data.jobs.length !== data.total_count) {
+ throw new Error('Grafana exporter supports at most 100 jobs; refusing a partial trace');
+ }
+ // Check the pinned exporter's name-based IDs and timestamp requirements.
+ const { createHash } = require('node:crypto');
+ const prefix = `${context.repo.owner}/${context.repo.repo}${run.id}${run.run_attempt}`;
+ const ids = new Set();
+ function span(suffix) {
+ const id = createHash('sha256').update(prefix + suffix).digest('hex').slice(16, 32);
+ if (ids.has(id)) throw new Error('Duplicate exporter span ID; job/step names must be distinct');
+ ids.add(id);
+ }
+ function timed(entry) {
+ const start = Date.parse(entry.started_at);
+ const end = Date.parse(entry.completed_at);
+ return entry.conclusion && Number.isFinite(start) && Number.isFinite(end) && end >= start;
+ }
+ span('s');
+ for (const job of data.jobs) {
+ if (job.status !== 'completed' || !timed(job)) {
+ throw new Error('Job timing is incomplete; retry after GitHub finalizes this attempt');
+ }
+ span(job.name);
+ for (const step of job.steps ?? []) {
+ if (!timed(step)) throw new Error('Step timing is unavailable; this exporter cannot represent it');
+ span(job.name + step.name);
+ }
+ }
+ // At 512 spans upstream starts an asynchronous batch whose failure can be swallowed.
+ if (ids.size >= 512) throw new Error('Grafana exporter cannot reliably report delivery failures for 512+ spans');
+ core.setOutput('export', data.jobs.length > 0);
+ if (!data.jobs.length) core.notice('No jobs in this attempt; no execution trace to export');
+ - name: Assume CI telemetry role
+ if: steps.preflight.outputs.export == 'true'
+ uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b
+ with:
+ role-to-assume: arn:aws:iam::569036502058:role/macro-observability-ci
+ aws-region: us-east-1
+ role-duration-seconds: '900'
+ - id: ingest
+ name: Read ingestion credential
+ if: steps.preflight.outputs.export == 'true'
+ run: |
+ set -euo pipefail
+ token=$(aws secretsmanager get-secret-value --region us-east-1 \
+ --secret-id observability/dev-ingest --query SecretString --output text)
+ if [[ ${#token} -lt 32 || "$token" == *[[:space:]]* ]]; then
+ echo 'Invalid ingestion credential' >&2
+ exit 1
+ fi
+ echo "::add-mask::$token"
+ headers=$(printf '%s' "$token" | python3 -c 'import sys, urllib.parse; print("Authorization=Bearer%20" + urllib.parse.quote(sys.stdin.read(), safe=""))')
+ echo "::add-mask::$headers"
+ printf 'headers=%s\n' "$headers" >> "$GITHUB_OUTPUT"
+ shell: bash
+ - id: traces
+ name: Export workflow, job and step traces
+ if: steps.preflight.outputs.export == 'true'
+ uses: grafana/grafana-github-actions/gha-otel-export@b1c422726cc4ff4adb0838ebca67586f24babb42
+ with:
+ github-token: ${{ github.token }}
+ env:
+ OTEL_SERVICE_NAME: github-actions
+ OTEL_RESOURCE_ATTRIBUTES: deployment.environment=ci
+ OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
+ OTEL_EXPORTER_OTLP_ENDPOINT: https://otlp.macro-internal.com
+ OTEL_EXPORTER_OTLP_HEADERS: ${{ steps.ingest.outputs.headers }}
diff --git a/infra/stacks/observability/Pulumi.prod.yaml b/infra/stacks/observability/Pulumi.prod.yaml
index c0379a62ba..51f9b03a4d 100644
--- a/infra/stacks/observability/Pulumi.prod.yaml
+++ b/infra/stacks/observability/Pulumi.prod.yaml
@@ -1,5 +1,6 @@
config:
aws:region: us-east-2
+ observability:ciIngestSecretArn: arn:aws:secretsmanager:us-east-1:569036502058:secret:observability/dev-ingest-7UhJbL
observability:alarmTopicArn: arn:aws:sns:us-east-2:569036502058:observability-prod-alerts
observability:amiId: ami-082c5befa76bfc246
observability:secretArn: arn:aws:secretsmanager:us-east-2:569036502058:secret:observability/prod-TwQpBJ
diff --git a/infra/stacks/observability/README.md b/infra/stacks/observability/README.md
index 1d8148acb6..14b0cc7a35 100644
--- a/infra/stacks/observability/README.md
+++ b/infra/stacks/observability/README.md
@@ -1,5 +1,66 @@
# Observability pilot
+## CI telemetry
+
+`tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs` defines
+this workflow. Run `cargo x workflows` to regenerate
+`.github/workflows/export-ci-telemetry.yml`; CI checks it for drift. The JavaScript
+preflight and shell credential reader are embedded from adjacent `scripts/` files
+at generation time, so runtime checkout is unnecessary.
+
+The generated workflow observes completed workflows on its
+explicit name list using Grafana's
+[`gha-otel-export`](https://github.com/grafana/grafana-github-actions/tree/b1c422726cc4ff4adb0838ebca67586f24babb42/gha-otel-export)
+action, pinned to that reviewed commit. It reads the triggering attempt's GitHub
+metadata and sends OTLP/HTTP protobuf traces to our existing endpoint. The trace
+hierarchy is workflow → jobs → steps, stored in Tempo with
+`service.name=github-actions` and `deployment.environment=ci`. There is no custom
+telemetry exporter, triggering PR checkout, artifact download, raw build-log
+collection, or Datadog export. Individual test cases are outside this integration.
+
+`/d/macro-ci` provides workflow traces, failed/cancelled jobs and steps, and jobs
+running longer than five minutes. Open a trace for its execution waterfall and
+GitHub URL. Search tables are limited to 100 traces and 100 matching spans per
+trace; narrow the time range to investigate further. These are bounded searches,
+not complete counts or p95 summaries. The action does not emit our old structured
+Loki events. Job durations exclude runner queue time. Workflow root duration
+includes waiting and, on reruns, starts at the original run creation time because
+of an upstream limitation; do not treat it as attempt execution time.
+
+### Pinned exporter limits
+
+A small metadata preflight rejects runs that the upstream action cannot export
+reliably: more than 100 jobs, 512 or more total workflow/job/step spans, colliding
+name-derived span IDs, or unavailable/invalid job or step timestamps. The action
+only fetches its first jobs page; its automatic batch export can swallow delivery
+errors at 512 spans. These guards catch those known cases before export.
+Skipped/cancelled jobs are included when GitHub supplies their timestamps; those
+without timing cause a visible exporter failure. Zero-job attempts are skipped
+with a notice. Remove limits only after upgrading and verifying the upstream fix.
+
+Deploy the prod Pulumi stack before enabling the workflow: it creates the account's
+GitHub OIDC provider and `macro-observability-ci` role, restricted to this repository's
+`main` identity and reading only `observability/dev-ingest` from Secrets Manager.
+There are no stored AWS access keys in this workflow. `ciIngestSecretArn` must match
+that existing Virginia secret. The bearer token is masked and passed only to the
+export step. The role can submit telemetry but cannot read stored traces or obtain
+Grafana's OAuth credentials. The provider is account-wide and protected; import it
+into Pulumi if another stack creates it before this rollout.
+
+Build, publish and smoke-boot a NixOS image with `ci-dashboard.nix`, then pin its AMI
+and deploy normally to provision the dashboard. The workflow starts observing after
+merge to `main`; add new workflow names to its list when they are introduced.
+GitHub's three-level `workflow_run` chaining limit still applies. Keep
+`trace-artifacts-glob` unset so the exporter never downloads PR-produced artifacts.
+
+Exporter failures appear as failed **Export CI telemetry** runs and do not change
+the observed workflow's result. Retry transient failures after GitHub finalizes job
+metadata. Deterministic trace IDs allow lookup across retries, but delivery is not
+transactional or guaranteed exactly once. Old spans are subject to Tempo's ingestion
+and retention limits. This is completion-time collection, with no live job progress,
+historical backfill, or CI alert notifications. Validate a successful and failed
+workflow, a rerun, and a multi-job trace after rollout before relying on the dashboard.
+
This stack starts Grafana alongside Datadog on one private EC2 instance. It
runs Grafana, Loki, Tempo, Prometheus, Alloy and nginx as native systemd services
on a prebuilt NixOS image. There is no Docker or Compose dependency. Nix owns
diff --git a/infra/stacks/observability/ci.ts b/infra/stacks/observability/ci.ts
new file mode 100644
index 0000000000..9869fed083
--- /dev/null
+++ b/infra/stacks/observability/ci.ts
@@ -0,0 +1,50 @@
+import * as aws from '@pulumi/aws';
+import * as pulumi from '@pulumi/pulumi';
+
+// Account-wide provider, owned only by the prod observability stack.
+export function createCiTelemetryRole(secretArn: string) {
+ const provider = new aws.iam.OpenIdConnectProvider(
+ 'github-actions',
+ {
+ url: 'https://token.actions.githubusercontent.com',
+ clientIdLists: ['sts.amazonaws.com'],
+ },
+ { protect: true }
+ );
+ const role = new aws.iam.Role('observability-ci', {
+ name: 'macro-observability-ci',
+ assumeRolePolicy: provider.arn.apply((arn) =>
+ JSON.stringify({
+ Version: '2012-10-17',
+ Statement: [
+ {
+ Effect: 'Allow',
+ Principal: { Federated: arn },
+ Action: 'sts:AssumeRoleWithWebIdentity',
+ Condition: {
+ StringEquals: {
+ 'token.actions.githubusercontent.com:aud': 'sts.amazonaws.com',
+ 'token.actions.githubusercontent.com:sub':
+ 'repo:macro-inc/macro:ref:refs/heads/main',
+ },
+ },
+ },
+ ],
+ })
+ ),
+ });
+ new aws.iam.RolePolicy('observability-ci-ingest', {
+ role: role.id,
+ policy: pulumi.jsonStringify({
+ Version: '2012-10-17',
+ Statement: [
+ {
+ Effect: 'Allow',
+ Action: 'secretsmanager:GetSecretValue',
+ Resource: secretArn,
+ },
+ ],
+ }),
+ });
+ return role.arn;
+}
diff --git a/infra/stacks/observability/index.ts b/infra/stacks/observability/index.ts
index 6ed1c0d3bd..b9e110520e 100644
--- a/infra/stacks/observability/index.ts
+++ b/infra/stacks/observability/index.ts
@@ -1,5 +1,6 @@
import * as aws from '@pulumi/aws';
import * as pulumi from '@pulumi/pulumi';
+import { createCiTelemetryRole } from './ci';
import { createNetwork } from './network';
import { renderUserData } from './render';
import {
@@ -40,6 +41,13 @@ validateSettings({
tracesBucket: 'validate-traces',
});
validateRegionalArn(alarmTopicArn, 'sns', region);
+const ciIngestSecretArn =
+ stack === 'prod' ? config.require('ciIngestSecretArn') : undefined;
+if (ciIngestSecretArn)
+ validateRegionalArn(ciIngestSecretArn, 'secretsmanager', 'us-east-1');
+export const ciTelemetryRoleArn = ciIngestSecretArn
+ ? createCiTelemetryRole(ciIngestSecretArn)
+ : undefined;
const vpc = createNetwork(region, tags);
const subnet = vpc.privateSubnet;
const subnetId = subnet.id;
diff --git a/infra/stacks/observability/nixos/ci-dashboard.nix b/infra/stacks/observability/nixos/ci-dashboard.nix
new file mode 100644
index 0000000000..b368c76e85
--- /dev/null
+++ b/infra/stacks/observability/nixos/ci-dashboard.nix
@@ -0,0 +1,113 @@
+{ pkgs, ... }:
+let
+ tempo = {
+ type = "tempo";
+ uid = "tempo";
+ };
+ selector = ''resource.service.name="github-actions" && resource.deployment.environment="ci"'';
+ panel = id: title: description: filter: fields: {
+ inherit id title description;
+ type = "table";
+ datasource = tempo;
+ gridPos = {
+ x = 0;
+ y = (id - 1) * 10;
+ w = 24;
+ h = 10;
+ };
+ targets = [
+ {
+ refId = "A";
+ queryType = "traceql";
+ query = "{ ${selector} && ${filter} } | select(name, ${fields})";
+ tableType = "spans";
+ limit = 100;
+ spss = 100;
+ }
+ ];
+ transformations = [
+ {
+ id = "organize";
+ options = {
+ excludeByName = {
+ "Trace Service" = true;
+ "deployment.environment" = true;
+ "service.name" = true;
+ workflow_run_id = true;
+ job_id = true;
+ workflow_name = true;
+ job_name = true;
+ step_name = true;
+ };
+ indexByName = {
+ "Name" = 0;
+ conclusion = 1;
+ "Duration" = 2;
+ "Start time" = 3;
+ "Span ID" = 4;
+ "Trace Name" = 5;
+ };
+ };
+ }
+ ];
+ options.showHeader = true;
+ };
+ dashboard = {
+ uid = "macro-ci";
+ title = "Macro CI";
+ description = "GitHub Actions workflow, job and step traces exported after completion. Select a trace to open its execution waterfall. Tables are bounded search results, not complete counts or percentile summaries. Narrow the time range when there are many runs.";
+ schemaVersion = 39;
+ version = 2;
+ editable = false;
+ tags = [
+ "macro"
+ "ci"
+ ];
+ timezone = "browser";
+ refresh = "1m";
+ time = {
+ from = "now-24h";
+ to = "now";
+ };
+ panels = [
+ (panel 1 "Workflow runs — open a trace for jobs and steps"
+ "Workflow duration includes waiting; for reruns the upstream exporter starts the root span at the original run creation. Use job durations for execution timing. The url attribute links to GitHub."
+ "span.workflow_run_id != nil"
+ "span.workflow_name, span.conclusion, span.run_attempt, span.url"
+ )
+ (panel 2 "Failed, cancelled or timed-out jobs"
+ "Cancellation is included; inspect the conclusion attribute to distinguish it from a failure."
+ "span.job_id != nil && status = error"
+ "span.workflow_name, span.job_name, span.conclusion, span.runner_name"
+ )
+ (panel 3 "Jobs taking more than five minutes"
+ "Job span duration measures execution, excluding runner queue time."
+ "span.job_id != nil && duration > 5m"
+ "span.workflow_name, span.job_name, span.conclusion, span.runner_name"
+ )
+ (panel 4 "Failed, cancelled or timed-out steps"
+ "Open the trace to see the step in its parent job and workflow. Raw build logs and individual test cases remain in GitHub."
+ "span.step_name != nil && status = error"
+ "span.step_name, span.conclusion"
+ )
+ ];
+ };
+in
+{
+ services.grafana.provision.dashboards.settings.providers = [
+ {
+ name = "macro-ci";
+ orgId = 1;
+ folder = "Macro";
+ type = "file";
+ disableDeletion = true;
+ allowUiUpdates = false;
+ options.path = pkgs.linkFarm "macro-ci-dashboard" [
+ {
+ name = "ci.json";
+ path = pkgs.writeText "ci.json" (builtins.toJSON dashboard);
+ }
+ ];
+ }
+ ];
+}
diff --git a/infra/stacks/observability/nixos/services.nix b/infra/stacks/observability/nixos/services.nix
index 3d05caf261..ed0f28e8b3 100644
--- a/infra/stacks/observability/nixos/services.nix
+++ b/infra/stacks/observability/nixos/services.nix
@@ -16,6 +16,7 @@ in
{
imports = [
./dashboards.nix
+ ./ci-dashboard.nix
./alerting.nix
];
services.grafana = {
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows.rs b/tooling/xtask/crates/xtask_workflows/src/workflows.rs
index 9a3c64905d..50a10dc740 100644
--- a/tooling/xtask/crates/xtask_workflows/src/workflows.rs
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows.rs
@@ -36,6 +36,7 @@ mod deploy_sync_service;
mod deploy_web_app;
mod deploy_website;
mod ensure_daytona_snapshot;
+mod export_ci_telemetry;
mod ios_preview;
mod path_validation;
mod publish_sdk;
@@ -105,6 +106,11 @@ fn job_mut<'a>(root: &'a mut serde_yaml::Value, id: &str) -> Result<&'a mut serd
/// Every workflow we generate. Add new workflows here.
const WORKFLOWS: &[WorkflowFile] = &[
+ WorkflowFile {
+ slug: "export_ci_telemetry",
+ file_name: "export-ci-telemetry.yml",
+ render_yaml: || render_gh_workflow(export_ci_telemetry::export_ci_telemetry)(),
+ },
WorkflowFile {
slug: "assign_author",
file_name: "assign-author.yml",
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs
new file mode 100644
index 0000000000..9683ee3b99
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs
@@ -0,0 +1,138 @@
+//! Export completed GitHub Actions attempts as workflow/job/step traces.
+
+use gh_workflow::{
+ Event, Expression, Job, Level, Permissions, Step, Workflow, WorkflowRun, WorkflowRunType,
+};
+
+// Explicit names avoid recursively observing this exporter. Add new workflows here.
+const OBSERVED_WORKFLOWS: &[&str] = &[
+ "Auto Label PRs",
+ "Build Agent Daemon on Tag",
+ "Build AppImage",
+ "Build Desktop on Tag",
+ "Build Rust CI Image",
+ "Build Web App",
+ "Build macOS DMG",
+ "CLA",
+ "Cancel Stuck Cloud Storage Deploys",
+ "Cleanup Preview",
+ "Deploy AI Editing Worker",
+ "Deploy All Services",
+ "Deploy CLA Worker",
+ "Deploy FusionAuth Instance",
+ "Deploy Lexical Service",
+ "Deploy Preview",
+ "Deploy Production Release",
+ "Deploy Service (Generic)",
+ "Deploy Sync Service",
+ "Deploy Web App",
+ "Deploy Website",
+ "Deploy analytics proxy dev",
+ "Deploy on Push",
+ "Ensure Daytona Snapshot",
+ "Export Datadog observability definitions",
+ "Migrate Macro Database",
+ "Publish Android to Google Play",
+ "Publish SDK",
+ "Pull Request",
+ "Pulumi Preview on PR",
+ "Push local stack binaries",
+ "Release SDK",
+ "Reusable Deploy Service",
+ "Reusable Preview Service",
+ "SDK Check",
+ "Web App Pr Checks",
+ "cargo deny check",
+ "cargo workspace dependency check",
+ "check generated workflows",
+ "check node_modules nix",
+ "cloud storage code check",
+ "conventions code check",
+ "deploy pulumi service",
+ "iOS Preview",
+ "infra code check",
+];
+const HAS_JOBS: &str = "steps.preflight.outputs.export == 'true'";
+
+/// Build the CI telemetry observer without checking out code or downloading artifacts.
+pub fn export_ci_telemetry() -> Workflow {
+ Workflow::new("Export CI telemetry")
+ .on(Event::default().workflow_run(
+ WorkflowRun::default()
+ .add_type(WorkflowRunType::Completed)
+ .workflows(
+ OBSERVED_WORKFLOWS
+ .iter()
+ .map(|name| (*name).to_owned())
+ .collect::<Vec<_>>(),
+ ),
+ ))
+ .permissions(Permissions {
+ contents: Some(Level::Read),
+ actions: Some(Level::Read),
+ id_token: Some(Level::Write),
+ ..Default::default()
+ })
+ .add_job("export", export())
+}
+
+fn export() -> Job {
+ Job::default()
+ // GitHub's hosted image supplies AWS CLI and Python without a checkout/setup step.
+ .runs_on("ubuntu-latest")
+ .timeout_minutes(10u32)
+ .add_step(
+ Step::new("Check upstream exporter limits")
+ .uses(
+ "actions",
+ "github-script",
+ "f28e40c7f34bde8b3046d885e986cb6290c5673b",
+ )
+ .id("preflight")
+ .add_with(("script", include_str!("scripts/ci_telemetry_preflight.js"))),
+ )
+ .add_step(
+ Step::new("Assume CI telemetry role")
+ .uses(
+ "aws-actions",
+ "configure-aws-credentials",
+ "e7f100cf4c008499ea8adda475de1042d6975c7b",
+ )
+ .if_condition(Expression::new(HAS_JOBS))
+ .add_with((
+ "role-to-assume",
+ "arn:aws:iam::569036502058:role/macro-observability-ci",
+ ))
+ .add_with(("aws-region", "us-east-1"))
+ .add_with(("role-duration-seconds", 900)),
+ )
+ .add_step(
+ Step::new("Read ingestion credential")
+ .run(include_str!("scripts/read_ci_ingestion_credential.sh"))
+ .id("ingest")
+ .if_condition(Expression::new(HAS_JOBS))
+ .shell("bash"),
+ )
+ .add_step(
+ Step::new("Export workflow, job and step traces")
+ .uses(
+ "grafana",
+ "grafana-github-actions/gha-otel-export",
+ "b1c422726cc4ff4adb0838ebca67586f24babb42",
+ )
+ .id("traces")
+ .if_condition(Expression::new(HAS_JOBS))
+ .add_with(("github-token", "${{ github.token }}"))
+ .add_env(("OTEL_SERVICE_NAME", "github-actions"))
+ .add_env(("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=ci"))
+ .add_env(("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"))
+ .add_env((
+ "OTEL_EXPORTER_OTLP_ENDPOINT",
+ "https://otlp.macro-internal.com",
+ ))
+ .add_env((
+ "OTEL_EXPORTER_OTLP_HEADERS",
+ "${{ steps.ingest.outputs.headers }}",
+ )),
+ )
+}
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_preflight.js b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_preflight.js
new file mode 100644
index 0000000000..278f28d720
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_preflight.js
@@ -0,0 +1,36 @@
+const run = context.payload.workflow_run;
+const { data } = await github.rest.actions.listJobsForWorkflowRunAttempt({
+ ...context.repo, run_id: run.id, attempt_number: run.run_attempt, per_page: 100,
+});
+if (data.total_count > 100 || data.jobs.length !== data.total_count) {
+ throw new Error('Grafana exporter supports at most 100 jobs; refusing a partial trace');
+}
+// Check the pinned exporter's name-based IDs and timestamp requirements.
+const { createHash } = require('node:crypto');
+const prefix = `${context.repo.owner}/${context.repo.repo}${run.id}${run.run_attempt}`;
+const ids = new Set();
+function span(suffix) {
+ const id = createHash('sha256').update(prefix + suffix).digest('hex').slice(16, 32);
+ if (ids.has(id)) throw new Error('Duplicate exporter span ID; job/step names must be distinct');
+ ids.add(id);
+}
+function timed(entry) {
+ const start = Date.parse(entry.started_at);
+ const end = Date.parse(entry.completed_at);
+ return entry.conclusion && Number.isFinite(start) && Number.isFinite(end) && end >= start;
+}
+span('s');
+for (const job of data.jobs) {
+ if (job.status !== 'completed' || !timed(job)) {
+ throw new Error('Job timing is incomplete; retry after GitHub finalizes this attempt');
+ }
+ span(job.name);
+ for (const step of job.steps ?? []) {
+ if (!timed(step)) throw new Error('Step timing is unavailable; this exporter cannot represent it');
+ span(job.name + step.name);
+ }
+}
+// At 512 spans upstream starts an asynchronous batch whose failure can be swallowed.
+if (ids.size >= 512) throw new Error('Grafana exporter cannot reliably report delivery failures for 512+ spans');
+core.setOutput('export', data.jobs.length > 0);
+if (!data.jobs.length) core.notice('No jobs in this attempt; no execution trace to export');
diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/read_ci_ingestion_credential.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/read_ci_ingestion_credential.sh
new file mode 100644
index 0000000000..1e3c0fa3ed
--- /dev/null
+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/read_ci_ingestion_credential.sh
@@ -0,0 +1,11 @@
+set -euo pipefail
+token=$(aws secretsmanager get-secret-value --region us-east-1 \
+ --secret-id observability/dev-ingest --query SecretString --output text)
+if [[ ${#token} -lt 32 || "$token" == *[[:space:]]* ]]; then
+ echo 'Invalid ingestion credential' >&2
+ exit 1
+fi
+echo "::add-mask::$token"
+headers=$(printf '%s' "$token" | python3 -c 'import sys, urllib.parse; print("Authorization=Bearer%20" + urllib.parse.quote(sys.stdin.read(), safe=""))')
+echo "::add-mask::$headers"
+printf 'headers=%s\n' "$headers" >> "$GITHUB_OUTPUT"