Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
chore(observability): export CI traces with Grafana action (#7883)
Wolf Mermelstein275ae30486fd
diff --git a/.github/workflows/export-ci-telemetry.yml b/.github/workflows/export-ci-telemetry.ymlnew file mode 100644index 0000000000..8aae61e606--- /dev/null+++ b/.github/workflows/export-ci-telemetry.yml@@ -0,0 +1,138 @@+# DO NOT EDIT — regenerate with `cargo x workflows` (from the repository root).+# Source: tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs+name: Export CI telemetry+'on':+ workflow_run:+ types:+ - completed+ workflows:+ - Auto Label PRs+ - Build Agent Daemon on Tag+ - Build AppImage+ - Build Desktop on Tag+ - Build Rust CI Image+ - Build Web App+ - Build macOS DMG+ - CLA+ - Cancel Stuck Cloud Storage Deploys+ - Cleanup Preview+ - Deploy AI Editing Worker+ - Deploy All Services+ - Deploy CLA Worker+ - Deploy FusionAuth Instance+ - Deploy Lexical Service+ - Deploy Preview+ - Deploy Production Release+ - Deploy Service (Generic)+ - Deploy Sync Service+ - Deploy Web App+ - Deploy Website+ - Deploy analytics proxy dev+ - Deploy on Push+ - Ensure Daytona Snapshot+ - Export Datadog observability definitions+ - Migrate Macro Database+ - Publish Android to Google Play+ - Publish SDK+ - Pull Request+ - Pulumi Preview on PR+ - Push local stack binaries+ - Release SDK+ - Reusable Deploy Service+ - Reusable Preview Service+ - SDK Check+ - Web App Pr Checks+ - cargo deny check+ - cargo workspace dependency check+ - check generated workflows+ - check node_modules nix+ - cloud storage code check+ - conventions code check+ - deploy pulumi service+ - iOS Preview+ - infra code check+permissions:+ actions: read+ contents: read+ id-token: write+jobs:+ export:+ runs-on: ubuntu-latest+ timeout-minutes: 10+ steps:+ - id: preflight+ name: Check upstream exporter limits+ uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b+ with:+ script: |+ const run = context.payload.workflow_run;+ const { data } = await github.rest.actions.listJobsForWorkflowRunAttempt({+ ...context.repo, run_id: run.id, attempt_number: run.run_attempt, per_page: 100,+ });+ if (data.total_count > 100 || data.jobs.length !== data.total_count) {+ throw new Error('Grafana exporter supports at most 100 jobs; refusing a partial trace');+ }+ // Check the pinned exporter's name-based IDs and timestamp requirements.+ const { createHash } = require('node:crypto');+ const prefix = `${context.repo.owner}/${context.repo.repo}${run.id}${run.run_attempt}`;+ const ids = new Set();+ function span(suffix) {+ const id = createHash('sha256').update(prefix + suffix).digest('hex').slice(16, 32);+ if (ids.has(id)) throw new Error('Duplicate exporter span ID; job/step names must be distinct');+ ids.add(id);+ }+ function timed(entry) {+ const start = Date.parse(entry.started_at);+ const end = Date.parse(entry.completed_at);+ return entry.conclusion && Number.isFinite(start) && Number.isFinite(end) && end >= start;+ }+ span('s');+ for (const job of data.jobs) {+ if (job.status !== 'completed' || !timed(job)) {+ throw new Error('Job timing is incomplete; retry after GitHub finalizes this attempt');+ }+ span(job.name);+ for (const step of job.steps ?? []) {+ if (!timed(step)) throw new Error('Step timing is unavailable; this exporter cannot represent it');+ span(job.name + step.name);+ }+ }+ // At 512 spans upstream starts an asynchronous batch whose failure can be swallowed.+ if (ids.size >= 512) throw new Error('Grafana exporter cannot reliably report delivery failures for 512+ spans');+ core.setOutput('export', data.jobs.length > 0);+ if (!data.jobs.length) core.notice('No jobs in this attempt; no execution trace to export');+ - name: Assume CI telemetry role+ if: steps.preflight.outputs.export == 'true'+ uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b+ with:+ role-to-assume: arn:aws:iam::569036502058:role/macro-observability-ci+ aws-region: us-east-1+ role-duration-seconds: '900'+ - id: ingest+ name: Read ingestion credential+ if: steps.preflight.outputs.export == 'true'+ run: |+ set -euo pipefail+ token=$(aws secretsmanager get-secret-value --region us-east-1 \+ --secret-id observability/dev-ingest --query SecretString --output text)+ if [[ ${#token} -lt 32 || "$token" == *[[:space:]]* ]]; then+ echo 'Invalid ingestion credential' >&2+ exit 1+ fi+ echo "::add-mask::$token"+ headers=$(printf '%s' "$token" | python3 -c 'import sys, urllib.parse; print("Authorization=Bearer%20" + urllib.parse.quote(sys.stdin.read(), safe=""))')+ echo "::add-mask::$headers"+ printf 'headers=%s\n' "$headers" >> "$GITHUB_OUTPUT"+ shell: bash+ - id: traces+ name: Export workflow, job and step traces+ if: steps.preflight.outputs.export == 'true'+ uses: grafana/grafana-github-actions/gha-otel-export@b1c422726cc4ff4adb0838ebca67586f24babb42+ with:+ github-token: ${{ github.token }}+ env:+ OTEL_SERVICE_NAME: github-actions+ OTEL_RESOURCE_ATTRIBUTES: deployment.environment=ci+ OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf+ OTEL_EXPORTER_OTLP_ENDPOINT: https://otlp.macro-internal.com+ OTEL_EXPORTER_OTLP_HEADERS: ${{ steps.ingest.outputs.headers }}diff --git a/infra/stacks/observability/Pulumi.prod.yaml b/infra/stacks/observability/Pulumi.prod.yamlindex c0379a62ba..51f9b03a4d 100644--- a/infra/stacks/observability/Pulumi.prod.yaml+++ b/infra/stacks/observability/Pulumi.prod.yaml@@ -1,5 +1,6 @@config:aws:region: us-east-2+ observability:ciIngestSecretArn: arn:aws:secretsmanager:us-east-1:569036502058:secret:observability/dev-ingest-7UhJbLobservability:alarmTopicArn: arn:aws:sns:us-east-2:569036502058:observability-prod-alertsobservability:amiId: ami-082c5befa76bfc246observability:secretArn: arn:aws:secretsmanager:us-east-2:569036502058:secret:observability/prod-TwQpBJdiff --git a/infra/stacks/observability/README.md b/infra/stacks/observability/README.mdindex 1d8148acb6..14b0cc7a35 100644--- a/infra/stacks/observability/README.md+++ b/infra/stacks/observability/README.md@@ -1,5 +1,66 @@# Observability pilot+## CI telemetry++`tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs` defines+this workflow. Run `cargo x workflows` to regenerate+`.github/workflows/export-ci-telemetry.yml`; CI checks it for drift. The JavaScript+preflight and shell credential reader are embedded from adjacent `scripts/` files+at generation time, so runtime checkout is unnecessary.++The generated workflow observes completed workflows on its+explicit name list using Grafana's+[`gha-otel-export`](https://github.com/grafana/grafana-github-actions/tree/b1c422726cc4ff4adb0838ebca67586f24babb42/gha-otel-export)+action, pinned to that reviewed commit. It reads the triggering attempt's GitHub+metadata and sends OTLP/HTTP protobuf traces to our existing endpoint. The trace+hierarchy is workflow → jobs → steps, stored in Tempo with+`service.name=github-actions` and `deployment.environment=ci`. There is no custom+telemetry exporter, triggering PR checkout, artifact download, raw build-log+collection, or Datadog export. Individual test cases are outside this integration.++`/d/macro-ci` provides workflow traces, failed/cancelled jobs and steps, and jobs+running longer than five minutes. Open a trace for its execution waterfall and+GitHub URL. Search tables are limited to 100 traces and 100 matching spans per+trace; narrow the time range to investigate further. These are bounded searches,+not complete counts or p95 summaries. The action does not emit our old structured+Loki events. Job durations exclude runner queue time. Workflow root duration+includes waiting and, on reruns, starts at the original run creation time because+of an upstream limitation; do not treat it as attempt execution time.++### Pinned exporter limits++A small metadata preflight rejects runs that the upstream action cannot export+reliably: more than 100 jobs, 512 or more total workflow/job/step spans, colliding+name-derived span IDs, or unavailable/invalid job or step timestamps. The action+only fetches its first jobs page; its automatic batch export can swallow delivery+errors at 512 spans. These guards catch those known cases before export.+Skipped/cancelled jobs are included when GitHub supplies their timestamps; those+without timing cause a visible exporter failure. Zero-job attempts are skipped+with a notice. Remove limits only after upgrading and verifying the upstream fix.++Deploy the prod Pulumi stack before enabling the workflow: it creates the account's+GitHub OIDC provider and `macro-observability-ci` role, restricted to this repository's+`main` identity and reading only `observability/dev-ingest` from Secrets Manager.+There are no stored AWS access keys in this workflow. `ciIngestSecretArn` must match+that existing Virginia secret. The bearer token is masked and passed only to the+export step. The role can submit telemetry but cannot read stored traces or obtain+Grafana's OAuth credentials. The provider is account-wide and protected; import it+into Pulumi if another stack creates it before this rollout.++Build, publish and smoke-boot a NixOS image with `ci-dashboard.nix`, then pin its AMI+and deploy normally to provision the dashboard. The workflow starts observing after+merge to `main`; add new workflow names to its list when they are introduced.+GitHub's three-level `workflow_run` chaining limit still applies. Keep+`trace-artifacts-glob` unset so the exporter never downloads PR-produced artifacts.++Exporter failures appear as failed **Export CI telemetry** runs and do not change+the observed workflow's result. Retry transient failures after GitHub finalizes job+metadata. Deterministic trace IDs allow lookup across retries, but delivery is not+transactional or guaranteed exactly once. Old spans are subject to Tempo's ingestion+and retention limits. This is completion-time collection, with no live job progress,+historical backfill, or CI alert notifications. Validate a successful and failed+workflow, a rerun, and a multi-job trace after rollout before relying on the dashboard.+This stack starts Grafana alongside Datadog on one private EC2 instance. Itruns Grafana, Loki, Tempo, Prometheus, Alloy and nginx as native systemd serviceson a prebuilt NixOS image. There is no Docker or Compose dependency. Nix ownsdiff --git a/infra/stacks/observability/ci.ts b/infra/stacks/observability/ci.tsnew file mode 100644index 0000000000..9869fed083--- /dev/null+++ b/infra/stacks/observability/ci.ts@@ -0,0 +1,50 @@+import * as aws from '@pulumi/aws';+import * as pulumi from '@pulumi/pulumi';++// Account-wide provider, owned only by the prod observability stack.+export function createCiTelemetryRole(secretArn: string) {+ const provider = new aws.iam.OpenIdConnectProvider(+ 'github-actions',+ {+ url: 'https://token.actions.githubusercontent.com',+ clientIdLists: ['sts.amazonaws.com'],+ },+ { protect: true }+ );+ const role = new aws.iam.Role('observability-ci', {+ name: 'macro-observability-ci',+ assumeRolePolicy: provider.arn.apply((arn) =>+ JSON.stringify({+ Version: '2012-10-17',+ Statement: [+ {+ Effect: 'Allow',+ Principal: { Federated: arn },+ Action: 'sts:AssumeRoleWithWebIdentity',+ Condition: {+ StringEquals: {+ 'token.actions.githubusercontent.com:aud': 'sts.amazonaws.com',+ 'token.actions.githubusercontent.com:sub':+ 'repo:macro-inc/macro:ref:refs/heads/main',+ },+ },+ },+ ],+ })+ ),+ });+ new aws.iam.RolePolicy('observability-ci-ingest', {+ role: role.id,+ policy: pulumi.jsonStringify({+ Version: '2012-10-17',+ Statement: [+ {+ Effect: 'Allow',+ Action: 'secretsmanager:GetSecretValue',+ Resource: secretArn,+ },+ ],+ }),+ });+ return role.arn;+}diff --git a/infra/stacks/observability/index.ts b/infra/stacks/observability/index.tsindex 6ed1c0d3bd..b9e110520e 100644--- a/infra/stacks/observability/index.ts+++ b/infra/stacks/observability/index.ts@@ -1,5 +1,6 @@import * as aws from '@pulumi/aws';import * as pulumi from '@pulumi/pulumi';+import { createCiTelemetryRole } from './ci';import { createNetwork } from './network';import { renderUserData } from './render';import {@@ -40,6 +41,13 @@ validateSettings({tracesBucket: 'validate-traces',});validateRegionalArn(alarmTopicArn, 'sns', region);+const ciIngestSecretArn =+ stack === 'prod' ? config.require('ciIngestSecretArn') : undefined;+if (ciIngestSecretArn)+ validateRegionalArn(ciIngestSecretArn, 'secretsmanager', 'us-east-1');+export const ciTelemetryRoleArn = ciIngestSecretArn+ ? createCiTelemetryRole(ciIngestSecretArn)+ : undefined;const vpc = createNetwork(region, tags);const subnet = vpc.privateSubnet;const subnetId = subnet.id;diff --git a/infra/stacks/observability/nixos/ci-dashboard.nix b/infra/stacks/observability/nixos/ci-dashboard.nixnew file mode 100644index 0000000000..b368c76e85--- /dev/null+++ b/infra/stacks/observability/nixos/ci-dashboard.nix@@ -0,0 +1,113 @@+{ pkgs, ... }:+let+ tempo = {+ type = "tempo";+ uid = "tempo";+ };+ selector = ''resource.service.name="github-actions" && resource.deployment.environment="ci"'';+ panel = id: title: description: filter: fields: {+ inherit id title description;+ type = "table";+ datasource = tempo;+ gridPos = {+ x = 0;+ y = (id - 1) * 10;+ w = 24;+ h = 10;+ };+ targets = [+ {+ refId = "A";+ queryType = "traceql";+ query = "{ ${selector} && ${filter} } | select(name, ${fields})";+ tableType = "spans";+ limit = 100;+ spss = 100;+ }+ ];+ transformations = [+ {+ id = "organize";+ options = {+ excludeByName = {+ "Trace Service" = true;+ "deployment.environment" = true;+ "service.name" = true;+ workflow_run_id = true;+ job_id = true;+ workflow_name = true;+ job_name = true;+ step_name = true;+ };+ indexByName = {+ "Name" = 0;+ conclusion = 1;+ "Duration" = 2;+ "Start time" = 3;+ "Span ID" = 4;+ "Trace Name" = 5;+ };+ };+ }+ ];+ options.showHeader = true;+ };+ dashboard = {+ uid = "macro-ci";+ title = "Macro CI";+ description = "GitHub Actions workflow, job and step traces exported after completion. Select a trace to open its execution waterfall. Tables are bounded search results, not complete counts or percentile summaries. Narrow the time range when there are many runs.";+ schemaVersion = 39;+ version = 2;+ editable = false;+ tags = [+ "macro"+ "ci"+ ];+ timezone = "browser";+ refresh = "1m";+ time = {+ from = "now-24h";+ to = "now";+ };+ panels = [+ (panel 1 "Workflow runs — open a trace for jobs and steps"+ "Workflow duration includes waiting; for reruns the upstream exporter starts the root span at the original run creation. Use job durations for execution timing. The url attribute links to GitHub."+ "span.workflow_run_id != nil"+ "span.workflow_name, span.conclusion, span.run_attempt, span.url"+ )+ (panel 2 "Failed, cancelled or timed-out jobs"+ "Cancellation is included; inspect the conclusion attribute to distinguish it from a failure."+ "span.job_id != nil && status = error"+ "span.workflow_name, span.job_name, span.conclusion, span.runner_name"+ )+ (panel 3 "Jobs taking more than five minutes"+ "Job span duration measures execution, excluding runner queue time."+ "span.job_id != nil && duration > 5m"+ "span.workflow_name, span.job_name, span.conclusion, span.runner_name"+ )+ (panel 4 "Failed, cancelled or timed-out steps"+ "Open the trace to see the step in its parent job and workflow. Raw build logs and individual test cases remain in GitHub."+ "span.step_name != nil && status = error"+ "span.step_name, span.conclusion"+ )+ ];+ };+in+{+ services.grafana.provision.dashboards.settings.providers = [+ {+ name = "macro-ci";+ orgId = 1;+ folder = "Macro";+ type = "file";+ disableDeletion = true;+ allowUiUpdates = false;+ options.path = pkgs.linkFarm "macro-ci-dashboard" [+ {+ name = "ci.json";+ path = pkgs.writeText "ci.json" (builtins.toJSON dashboard);+ }+ ];+ }+ ];+}diff --git a/infra/stacks/observability/nixos/services.nix b/infra/stacks/observability/nixos/services.nixindex 3d05caf261..ed0f28e8b3 100644--- a/infra/stacks/observability/nixos/services.nix+++ b/infra/stacks/observability/nixos/services.nix@@ -16,6 +16,7 @@ in{imports = [./dashboards.nix+ ./ci-dashboard.nix./alerting.nix];services.grafana = {diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows.rs b/tooling/xtask/crates/xtask_workflows/src/workflows.rsindex 9a3c64905d..50a10dc740 100644--- a/tooling/xtask/crates/xtask_workflows/src/workflows.rs+++ b/tooling/xtask/crates/xtask_workflows/src/workflows.rs@@ -36,6 +36,7 @@ mod deploy_sync_service;mod deploy_web_app;mod deploy_website;mod ensure_daytona_snapshot;+mod export_ci_telemetry;mod ios_preview;mod path_validation;mod publish_sdk;@@ -105,6 +106,11 @@ fn job_mut<'a>(root: &'a mut serde_yaml::Value, id: &str) -> Result<&'a mut serd/// Every workflow we generate. Add new workflows here.const WORKFLOWS: &[WorkflowFile] = &[+ WorkflowFile {+ slug: "export_ci_telemetry",+ file_name: "export-ci-telemetry.yml",+ render_yaml: || render_gh_workflow(export_ci_telemetry::export_ci_telemetry)(),+ },WorkflowFile {slug: "assign_author",file_name: "assign-author.yml",diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rsnew file mode 100644index 0000000000..9683ee3b99--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/export_ci_telemetry.rs@@ -0,0 +1,138 @@+//! Export completed GitHub Actions attempts as workflow/job/step traces.++use gh_workflow::{+ Event, Expression, Job, Level, Permissions, Step, Workflow, WorkflowRun, WorkflowRunType,+};++// Explicit names avoid recursively observing this exporter. Add new workflows here.+const OBSERVED_WORKFLOWS: &[&str] = &[+ "Auto Label PRs",+ "Build Agent Daemon on Tag",+ "Build AppImage",+ "Build Desktop on Tag",+ "Build Rust CI Image",+ "Build Web App",+ "Build macOS DMG",+ "CLA",+ "Cancel Stuck Cloud Storage Deploys",+ "Cleanup Preview",+ "Deploy AI Editing Worker",+ "Deploy All Services",+ "Deploy CLA Worker",+ "Deploy FusionAuth Instance",+ "Deploy Lexical Service",+ "Deploy Preview",+ "Deploy Production Release",+ "Deploy Service (Generic)",+ "Deploy Sync Service",+ "Deploy Web App",+ "Deploy Website",+ "Deploy analytics proxy dev",+ "Deploy on Push",+ "Ensure Daytona Snapshot",+ "Export Datadog observability definitions",+ "Migrate Macro Database",+ "Publish Android to Google Play",+ "Publish SDK",+ "Pull Request",+ "Pulumi Preview on PR",+ "Push local stack binaries",+ "Release SDK",+ "Reusable Deploy Service",+ "Reusable Preview Service",+ "SDK Check",+ "Web App Pr Checks",+ "cargo deny check",+ "cargo workspace dependency check",+ "check generated workflows",+ "check node_modules nix",+ "cloud storage code check",+ "conventions code check",+ "deploy pulumi service",+ "iOS Preview",+ "infra code check",+];+const HAS_JOBS: &str = "steps.preflight.outputs.export == 'true'";++/// Build the CI telemetry observer without checking out code or downloading artifacts.+pub fn export_ci_telemetry() -> Workflow {+ Workflow::new("Export CI telemetry")+ .on(Event::default().workflow_run(+ WorkflowRun::default()+ .add_type(WorkflowRunType::Completed)+ .workflows(+ OBSERVED_WORKFLOWS+ .iter()+ .map(|name| (*name).to_owned())+ .collect::<Vec<_>>(),+ ),+ ))+ .permissions(Permissions {+ contents: Some(Level::Read),+ actions: Some(Level::Read),+ id_token: Some(Level::Write),+ ..Default::default()+ })+ .add_job("export", export())+}++fn export() -> Job {+ Job::default()+ // GitHub's hosted image supplies AWS CLI and Python without a checkout/setup step.+ .runs_on("ubuntu-latest")+ .timeout_minutes(10u32)+ .add_step(+ Step::new("Check upstream exporter limits")+ .uses(+ "actions",+ "github-script",+ "f28e40c7f34bde8b3046d885e986cb6290c5673b",+ )+ .id("preflight")+ .add_with(("script", include_str!("scripts/ci_telemetry_preflight.js"))),+ )+ .add_step(+ Step::new("Assume CI telemetry role")+ .uses(+ "aws-actions",+ "configure-aws-credentials",+ "e7f100cf4c008499ea8adda475de1042d6975c7b",+ )+ .if_condition(Expression::new(HAS_JOBS))+ .add_with((+ "role-to-assume",+ "arn:aws:iam::569036502058:role/macro-observability-ci",+ ))+ .add_with(("aws-region", "us-east-1"))+ .add_with(("role-duration-seconds", 900)),+ )+ .add_step(+ Step::new("Read ingestion credential")+ .run(include_str!("scripts/read_ci_ingestion_credential.sh"))+ .id("ingest")+ .if_condition(Expression::new(HAS_JOBS))+ .shell("bash"),+ )+ .add_step(+ Step::new("Export workflow, job and step traces")+ .uses(+ "grafana",+ "grafana-github-actions/gha-otel-export",+ "b1c422726cc4ff4adb0838ebca67586f24babb42",+ )+ .id("traces")+ .if_condition(Expression::new(HAS_JOBS))+ .add_with(("github-token", "${{ github.token }}"))+ .add_env(("OTEL_SERVICE_NAME", "github-actions"))+ .add_env(("OTEL_RESOURCE_ATTRIBUTES", "deployment.environment=ci"))+ .add_env(("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"))+ .add_env((+ "OTEL_EXPORTER_OTLP_ENDPOINT",+ "https://otlp.macro-internal.com",+ ))+ .add_env((+ "OTEL_EXPORTER_OTLP_HEADERS",+ "${{ steps.ingest.outputs.headers }}",+ )),+ )+}diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_preflight.js b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_preflight.jsnew file mode 100644index 0000000000..278f28d720--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/ci_telemetry_preflight.js@@ -0,0 +1,36 @@+const run = context.payload.workflow_run;+const { data } = await github.rest.actions.listJobsForWorkflowRunAttempt({+ ...context.repo, run_id: run.id, attempt_number: run.run_attempt, per_page: 100,+});+if (data.total_count > 100 || data.jobs.length !== data.total_count) {+ throw new Error('Grafana exporter supports at most 100 jobs; refusing a partial trace');+}+// Check the pinned exporter's name-based IDs and timestamp requirements.+const { createHash } = require('node:crypto');+const prefix = `${context.repo.owner}/${context.repo.repo}${run.id}${run.run_attempt}`;+const ids = new Set();+function span(suffix) {+ const id = createHash('sha256').update(prefix + suffix).digest('hex').slice(16, 32);+ if (ids.has(id)) throw new Error('Duplicate exporter span ID; job/step names must be distinct');+ ids.add(id);+}+function timed(entry) {+ const start = Date.parse(entry.started_at);+ const end = Date.parse(entry.completed_at);+ return entry.conclusion && Number.isFinite(start) && Number.isFinite(end) && end >= start;+}+span('s');+for (const job of data.jobs) {+ if (job.status !== 'completed' || !timed(job)) {+ throw new Error('Job timing is incomplete; retry after GitHub finalizes this attempt');+ }+ span(job.name);+ for (const step of job.steps ?? []) {+ if (!timed(step)) throw new Error('Step timing is unavailable; this exporter cannot represent it');+ span(job.name + step.name);+ }+}+// At 512 spans upstream starts an asynchronous batch whose failure can be swallowed.+if (ids.size >= 512) throw new Error('Grafana exporter cannot reliably report delivery failures for 512+ spans');+core.setOutput('export', data.jobs.length > 0);+if (!data.jobs.length) core.notice('No jobs in this attempt; no execution trace to export');diff --git a/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/read_ci_ingestion_credential.sh b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/read_ci_ingestion_credential.shnew file mode 100644index 0000000000..1e3c0fa3ed--- /dev/null+++ b/tooling/xtask/crates/xtask_workflows/src/workflows/scripts/read_ci_ingestion_credential.sh@@ -0,0 +1,11 @@+set -euo pipefail+token=$(aws secretsmanager get-secret-value --region us-east-1 \+ --secret-id observability/dev-ingest --query SecretString --output text)+if [[ ${#token} -lt 32 || "$token" == *[[:space:]]* ]]; then+ echo 'Invalid ingestion credential' >&2+ exit 1+fi+echo "::add-mask::$token"+headers=$(printf '%s' "$token" | python3 -c 'import sys, urllib.parse; print("Authorization=Bearer%20" + urllib.parse.quote(sys.stdin.read(), safe=""))')+echo "::add-mask::$headers"+printf 'headers=%s\n' "$headers" >> "$GITHUB_OUTPUT"