Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Duplicate dev telemetry to Grafana while preserving direct Datadog export (#7863)
Wolf Mermelstein67abe0b87fcd
diff --git a/.github/workflows/deploy-analytics-proxy-dev.yml b/.github/workflows/deploy-analytics-proxy-dev.ymlnew file mode 100644index 0000000000..99ccb33cdc--- /dev/null+++ b/.github/workflows/deploy-analytics-proxy-dev.yml@@ -0,0 +1,47 @@+name: Deploy analytics proxy dev+on:+ push:+ branches: [main]+ paths:+ - services/analytics-proxy/**+ - .github/workflows/deploy-analytics-proxy-dev.yml+ workflow_dispatch:+permissions:+ contents: read+concurrency:+ group: deploy-analytics-proxy-dev+ cancel-in-progress: false+jobs:+ deploy:+ runs-on: namespace-profile-linux-small+ defaults:+ run:+ working-directory: services/analytics-proxy+ steps:+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10+ with:+ persist-credentials: false+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020+ with:+ node-version: '22'+ - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b+ with:+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}+ aws-region: us-east-1+ - run: bun install --frozen-lockfile+ - run: bun run gen-types && bun run check+ - name: Set server-side ingestion credential+ env:+ CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}+ CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}+ run: |+ set -euo pipefail+ aws secretsmanager get-secret-value --secret-id observability/dev-ingest --query SecretString --output text |+ bunx wrangler secret put GRAFANA_OTLP_TOKEN --env dev+ - name: Deploy dev proxy+ run: bun run deploy-dev+ env:+ CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}+ CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}diff --git a/.github/workflows/export-datadog-observability.yml b/.github/workflows/export-datadog-observability.ymlnew file mode 100644index 0000000000..7e28452512--- /dev/null+++ b/.github/workflows/export-datadog-observability.yml@@ -0,0 +1,24 @@+name: Export Datadog observability definitions+on:+ workflow_dispatch:+permissions:+ contents: read+jobs:+ export:+ runs-on: namespace-profile-linux-small+ steps:+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10+ with:+ persist-credentials: false+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6+ - name: Read dashboards and monitors+ env:+ DD_API_KEY: ${{ secrets.DD_API_KEY }}+ DD_APP_KEY: ${{ secrets.DD_APP_KEY }}+ run: bun infra/stacks/observability/migration/export-datadog.ts /tmp/datadog-inventory+ - if: always()+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02+ with:+ name: datadog-observability-definitions+ path: /tmp/datadog-inventory+ retention-days: 7diff --git a/crates/macro_entrypoint/src/grafana.rs b/crates/macro_entrypoint/src/grafana.rsnew file mode 100644index 0000000000..9830b11d4a--- /dev/null+++ b/crates/macro_entrypoint/src/grafana.rs@@ -0,0 +1,35 @@+//! Failure isolation for the optional Grafana trace copy.++use opentelemetry_sdk::{+ Resource,+ error::OTelSdkResult,+ trace::{SpanData, SpanExporter},+};+use std::time::Duration;++/// Drops failed optional copies without producing SDK application-error logs in Datadog.+#[derive(Debug)]+pub(crate) struct GrafanaSpanExporter(pub(crate) opentelemetry_otlp::SpanExporter);++impl SpanExporter for GrafanaSpanExporter {+ async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {+ // The inner exporter has a two-second timeout. Returning its failure would+ // make BatchSpanProcessor log ERROR through the application's Datadog sink.+ let _ = self.0.export(batch).await;+ Ok(())+ }++ fn set_resource(&mut self, resource: &Resource) {+ self.0.set_resource(resource);+ }++ fn shutdown_with_timeout(&mut self, timeout: Duration) -> OTelSdkResult {+ let _ = self.0.shutdown_with_timeout(timeout);+ Ok(())+ }++ fn force_flush(&mut self) -> OTelSdkResult {+ let _ = self.0.force_flush();+ Ok(())+ }+}diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rsindex 4d3a9de367..b83ba3ae13 100644--- a/crates/macro_entrypoint/src/lib.rs+++ b/crates/macro_entrypoint/src/lib.rs@@ -3,6 +3,7 @@//! This is used to provide consistent behaviour with e.g. tracing configurationsmod datadog_fmt;+mod grafana;mod shutdown;#[cfg(test)]@@ -36,6 +37,7 @@ maybe_env_vars! {pub struct RustLog;pub struct OtelExporterOtlpEndpoint;pub struct OtelTraceFilter;+ pub struct GrafanaOtlpEndpoint;}/// Build an [`EnvFilter`] from `RUST_LOG`, honoring values injected via `APP_SECRETS_JSON`.@@ -167,7 +169,8 @@ impl MacroEntrypoint {// LGTM stack), so local logs are queryable next to the traces// they belong to.let export_otel = OtelExporterOtlpEndpoint::new().is_some();- let tracer_provider = export_otel.then(|| init_opentelemetry("local".to_string()));+ let tracer_provider =+ export_otel.then(|| init_opentelemetry("local".to_string(), None));let logger_provider = export_otel.then(|| init_otel_logs("local".to_string()));if let Some(provider) = tracer_provider.as_ref() {@@ -241,7 +244,10 @@ impl MacroEntrypoint {let env = DdEnv::new().map(|e| e.to_string()).unwrap_or_else(|_| "unknown".to_string());- let tracer_provider = init_opentelemetry(env);+ let grafana_endpoint = matches!(self.env, Environment::Develop)+ .then(GrafanaOtlpEndpoint::new)+ .flatten();+ let tracer_provider = init_opentelemetry(env, grafana_endpoint.as_deref());let tracer = tracer_provider.tracer(service_name());let rust_log_filter = rust_log_env_filter();@@ -350,7 +356,10 @@ fn otel_resource(deployment_environment: String) -> opentelemetry_sdk::Resource.build()}-fn init_opentelemetry(deployment_environment: String) -> SdkTracerProvider {+fn init_opentelemetry(+ deployment_environment: String,+ grafana_endpoint: Option<&str>,+) -> SdkTracerProvider {// W3C trace-context propagation: lets macro_tower_layers parent request// spans under an incoming `traceparent` (e.g. from the web app), and// service clients propagate context onward.@@ -364,10 +373,40 @@ fn init_opentelemetry(deployment_environment: String) -> SdkTracerProvider {.build().expect("failed to create OTLP span exporter");- SdkTracerProvider::builder()+ let mut provider = SdkTracerProvider::builder().with_batch_exporter(exporter)- .with_resource(otel_resource(deployment_environment))- .build()+ .with_resource(otel_resource(deployment_environment));++ // Keep the original exporter and its queue independent. The second batch+ // processor copies the same finished spans without changing IDs or sampling.+ if let Some(endpoint) = grafana_endpoint.filter(|endpoint| !endpoint.trim().is_empty()) {+ match opentelemetry_otlp::SpanExporter::builder()+ .with_tonic()+ .with_endpoint(endpoint)+ .with_timeout(std::time::Duration::from_secs(2))+ .build()+ {+ Ok(exporter) => {+ // At most one in-flight and one queued batch: two two-second+ // exports fit within the SDK's five-second shutdown timeout.+ let processor = opentelemetry_sdk::trace::BatchSpanProcessor::builder(+ grafana::GrafanaSpanExporter(exporter),+ )+ .with_batch_config(+ opentelemetry_sdk::trace::BatchConfigBuilder::default()+ .with_max_queue_size(512)+ .with_max_export_batch_size(512)+ .build(),+ )+ .build();+ provider = provider.with_span_processor(processor);+ }+ // Tracing is not installed yet. Optional export must not prevent startup.+ Err(error) => eprintln!("Grafana trace exporter unavailable: {error}"),+ }+ }++ provider.build()}/// OTLP log export for the [`OpenTelemetryTracingBridge`]: tracing eventsdiff --git a/infra/packages/resources/src/index.ts b/infra/packages/resources/src/index.tsindex 68aeeef900..05ee70d021 100644--- a/infra/packages/resources/src/index.ts+++ b/infra/packages/resources/src/index.ts@@ -37,3 +37,5 @@ export {export type { TopicArgs } from './resources/kafka-cluster';export { KafkaCluster } from './resources/kafka-cluster';export { ServiceTargetGroup } from './resources/service_target_group';++export { withTelemetry } from './resources/telemetry';diff --git a/infra/packages/resources/src/resources/datadog.ts b/infra/packages/resources/src/resources/datadog.tsindex aa04713b28..c604176d08 100644--- a/infra/packages/resources/src/resources/datadog.ts+++ b/infra/packages/resources/src/resources/datadog.ts@@ -5,7 +5,7 @@ import {ComponentResource,type ComponentResourceOptions,} from '@pulumi/pulumi';-import { stack } from '../../../shared';+import { grafanaTelemetryEnabled, stack } from '../../../shared';const DATADOG_API_KEY_SECRET_KEY = 'datadog-api-key';@@ -78,7 +78,9 @@ export const fargateLogRouterSidecarContainer = {type: 'fluentbit',options: {'config-file-type': 'file',- 'config-file-value': '/fluent-bit/configs/parse-json.conf',+ 'config-file-value': grafanaTelemetryEnabled+ ? '/tmp/observability-extra.conf'+ : '/fluent-bit/configs/parse-json.conf','enable-ecs-log-metadata': 'true',},},@@ -96,6 +98,24 @@ export const fargateLogRouterSidecarContainer = {value: stack,},],+ ...(grafanaTelemetryEnabled+ ? {+ entryPoint: ['/bin/sh', '-ec'],+ command: [+ `cat /fluent-bit/configs/parse-json.conf > /tmp/observability-extra.conf+cat >> /tmp/observability-extra.conf <<'CONFIG'+[OUTPUT]+ Name forward+ Match *-firelens-*+ Host 127.0.0.1+ Port 24225+ Retry_Limit 1+ net.connect_timeout 2+CONFIG+exec /entrypoint.sh`,+ ],+ }+ : {}),memoryReservation: 50,} satisfies ecs.TaskDefinitionContainerDefinitionArgs;diff --git a/infra/packages/resources/src/resources/ecs-telemetry.alloy b/infra/packages/resources/src/resources/ecs-telemetry.alloynew file mode 100644index 0000000000..b0aa381429--- /dev/null+++ b/infra/packages/resources/src/resources/ecs-telemetry.alloy@@ -0,0 +1,103 @@+logging { level = "warn" }++otelcol.receiver.otlp "applications" {+ grpc { endpoint = "127.0.0.1:14317" }+ http { endpoint = "127.0.0.1:14318" }+ output {+ traces = [otelcol.processor.batch.grafana.input]+ metrics = [otelcol.processor.batch.grafana.input]+ logs = [otelcol.processor.batch.grafana.input]+ }+}++otelcol.receiver.fluentforward "application_logs" {+ endpoint = "127.0.0.1:24225"+ output { logs = [otelcol.processor.batch.grafana.input] }+}++otelcol.receiver.awsecscontainermetrics "task" {+ collection_interval = "60s"+ output { metrics = [otelcol.processor.batch.grafana.input] }+}++otelcol.processor.memory_limiter "grafana" {+ check_interval = "1s"+ limit = "384MiB"+ spike_limit = "64MiB"+ output {+ logs = [otelcol.processor.transform.grafana.input]+ traces = [otelcol.processor.transform.grafana.input]+ metrics = [otelcol.processor.transform.grafana.input]+ }+}++otelcol.processor.transform "grafana" {+ error_mode = "ignore"+ log_statements {+ context = "log"+ statements = [+ `set(resource.attributes["service.name"], "` + sys.env("TELEMETRY_SERVICE") + `") where resource.attributes["service.name"] == nil`,+ `set(resource.attributes["deployment.environment.name"], "dev")`,+ `set(resource.attributes["deployment.environment"], "dev")`,+ `set(attributes["container_name"], body["container_name"]) where IsMap(body)`,+ `set(attributes["ecs_task_arn"], body["ecs_task_arn"]) where IsMap(body)`,+ `set(body, body["log"]) where IsMap(body) and body["log"] != nil`,+ ]+ }+ trace_statements {+ context = "resource"+ statements = [+ `set(attributes["deployment.environment.name"], "dev")`,+ `set(attributes["deployment.environment"], "dev")`,+ ]+ }+ metric_statements {+ context = "resource"+ statements = [+ `set(attributes["service.name"], "` + sys.env("TELEMETRY_SERVICE") + `") where attributes["service.name"] == nil`,+ `set(attributes["deployment.environment.name"], "dev")`,+ `set(attributes["deployment.environment"], "dev")`,+ ]+ }+ // Preserve series identity when the destination converts OTLP to Prometheus.+ metric_statements {+ context = "datapoint"+ statements = [+ `set(attributes["deployment.environment"], "dev")`,+ `set(attributes["aws.ecs.task.arn"], resource.attributes["aws.ecs.task.arn"]) where resource.attributes["aws.ecs.task.arn"] != nil`,+ `set(attributes["container.id"], resource.attributes["container.id"]) where resource.attributes["container.id"] != nil`,+ `set(attributes["container.name"], resource.attributes["container.name"]) where resource.attributes["container.name"] != nil`,+ ]+ }+ output {+ logs = [otelcol.exporter.otlphttp.grafana.input]+ traces = [otelcol.exporter.otlphttp.grafana.input]+ metrics = [otelcol.exporter.otlphttp.grafana.input]+ }+}++// Buffer only the independent Grafana copy; Datadog bypasses this collector.+otelcol.processor.batch "grafana" {+ timeout = "5s"+ send_batch_size = 256+ send_batch_max_size = 512+ output {+ logs = [otelcol.processor.memory_limiter.grafana.input]+ traces = [otelcol.processor.memory_limiter.grafana.input]+ metrics = [otelcol.processor.memory_limiter.grafana.input]+ }+}++otelcol.auth.bearer "grafana" {+ token = sys.env("GRAFANA_OTLP_TOKEN")+}++otelcol.exporter.otlphttp "grafana" {+ client {+ endpoint = "https://otlp.macro-internal.com"+ auth = otelcol.auth.bearer.grafana.handler+ timeout = "5s"+ }+ sending_queue { queue_size = 128 }+ retry_on_failure { max_elapsed_time = "60s" }+}diff --git a/infra/packages/resources/src/resources/grafana.ts b/infra/packages/resources/src/resources/grafana.tsnew file mode 100644index 0000000000..9aabba8d13--- /dev/null+++ b/infra/packages/resources/src/resources/grafana.ts@@ -0,0 +1,45 @@+import { readFileSync } from 'node:fs';+import { join } from 'node:path';+import type { ecs } from '@pulumi/awsx/types/input';+import {+ grafanaIngestSecretArn,+ grafanaTelemetryEnabled,+} from '../../../shared';++// Applied automatically with the dev sidecar; never overrides the primary OTLP endpoint.+export const grafanaTelemetryEnvironment = grafanaTelemetryEnabled+ ? [{ name: 'GRAFANA_OTLP_ENDPOINT', value: 'http://127.0.0.1:14317' }]+ : [];++export function grafanaTelemetryContainers(+ serviceName: string+): Record<string, ecs.TaskDefinitionContainerDefinitionArgs> {+ if (!grafanaTelemetryEnabled || !grafanaIngestSecretArn) return {};+ return {+ alloy: {+ name: 'alloy',+ image:+ 'grafana/alloy:v1.16.0@sha256:6e00cf7c5a692ff5f24844529416ed017d76fce922f8199004e73d5eca46b6b8',+ essential: false,+ memory: 512,+ memoryReservation: 128,+ stopTimeout: 30,+ user: '473',+ entryPoint: ['/bin/sh', '-ec'],+ command: [+ 'umask 077; printf "%s" "$ALLOY_CONFIG" > /tmp/config.alloy; exec /bin/alloy run --stability.level=experimental --server.http.listen-addr=127.0.0.1:12345 --storage.path=/tmp/alloy /tmp/config.alloy',+ ],+ environment: [+ {+ name: 'ALLOY_CONFIG',+ value: readFileSync(join(__dirname, 'ecs-telemetry.alloy'), 'utf8'),+ },+ { name: 'TELEMETRY_SERVICE', value: serviceName },+ { name: 'GOMEMLIMIT', value: '400MiB' },+ ],+ secrets: [+ { name: 'GRAFANA_OTLP_TOKEN', valueFrom: grafanaIngestSecretArn },+ ],+ },+ };+}diff --git a/infra/packages/resources/src/resources/telemetry.ts b/infra/packages/resources/src/resources/telemetry.tsnew file mode 100644index 0000000000..bcf52fb300--- /dev/null+++ b/infra/packages/resources/src/resources/telemetry.ts@@ -0,0 +1,49 @@+import type { ecs } from '@pulumi/awsx/types/input';+import * as pulumi from '@pulumi/pulumi';+import { grafanaTelemetryEnabled } from '../../../shared';+import {+ datadogAgentContainer,+ fargateLogRouterSidecarContainer,+} from './datadog';+import {+ grafanaTelemetryContainers,+ grafanaTelemetryEnvironment,+} from './grafana';++/** Standard app container and telemetry sidecars, with Grafana copies in dev. */+export function withTelemetry(+ serviceName: string,+ containers: Record<string, ecs.TaskDefinitionContainerDefinitionArgs>,+ sidecars: {+ logRouter?: Partial<ecs.TaskDefinitionContainerDefinitionArgs>;+ datadogAgent?: Partial<ecs.TaskDefinitionContainerDefinitionArgs>;+ } = {}+): Record<string, ecs.TaskDefinitionContainerDefinitionArgs> {+ return {+ ...grafanaTelemetryContainers(serviceName),+ log_router: {+ ...fargateLogRouterSidecarContainer,+ ...sidecars.logRouter,+ },+ datadog_agent: {+ ...datadogAgentContainer,+ ...sidecars.datadogAgent,+ },+ ...Object.fromEntries(+ Object.entries(containers).map(([name, container]) => [+ name,+ grafanaTelemetryEnabled+ ? {+ ...container,+ environment: pulumi+ .output(container.environment)+ .apply((environment) => [+ ...(environment ?? []),+ ...grafanaTelemetryEnvironment,+ ]),+ }+ : container,+ ])+ ),+ };+}diff --git a/infra/packages/shared/src/doppler_environment.ts b/infra/packages/shared/src/doppler_environment.tsindex 36df7f804b..9202a15c7f 100644--- a/infra/packages/shared/src/doppler_environment.ts+++ b/infra/packages/shared/src/doppler_environment.ts@@ -1,6 +1,7 @@import * as aws from '@pulumi/aws';import * as pulumi from '@pulumi/pulumi';import { stack } from '.';+import { grafanaIngestSecretArn } from './grafana_telemetry';export type DopplerEcsEnvironmentArgs = {// Resource tags@@ -35,7 +36,9 @@ export class DopplerEcsEnvironment extends pulumi.ComponentResource {Statement: [{Action: ['secretsmanager:GetSecretValue'],- Resource: pulumi.interpolate`${dopplerSecretSyncArn}`,+ Resource: grafanaIngestSecretArn+ ? [dopplerSecretSyncArn, grafanaIngestSecretArn]+ : pulumi.interpolate`${dopplerSecretSyncArn}`,Effect: 'Allow',},],diff --git a/infra/packages/shared/src/grafana_telemetry.ts b/infra/packages/shared/src/grafana_telemetry.tsnew file mode 100644index 0000000000..703db14b27--- /dev/null+++ b/infra/packages/shared/src/grafana_telemetry.ts@@ -0,0 +1,10 @@+import * as aws from '@pulumi/aws';+import * as pulumi from '@pulumi/pulumi';++// All dev ECS services run the shared telemetry sidecars. Production follows in a later rollout.+export const grafanaTelemetryEnabled = pulumi.getStack() === 'dev';++// Only the ingestion token is stored here; never grant tasks the Grafana OAuth bundle.+export const grafanaIngestSecretArn = grafanaTelemetryEnabled+ ? aws.secretsmanager.getSecretOutput({ name: 'observability/dev-ingest' }).arn+ : undefined;diff --git a/infra/packages/shared/src/index.ts b/infra/packages/shared/src/index.tsindex 3b2ea74b5f..26d85fa4ee 100644--- a/infra/packages/shared/src/index.ts+++ b/infra/packages/shared/src/index.ts@@ -72,3 +72,8 @@ export { getGatewayAlb, type GatewayAlb } from './gateway';export { GATEWAY_PRIORITIES, GatewayService } from './gateway_priorities';export * from './service_urls';++export {+ grafanaTelemetryEnabled,+ grafanaIngestSecretArn,+} from './grafana_telemetry';diff --git a/infra/stacks/agent-harness-service/agent_harness_service.ts b/infra/stacks/agent-harness-service/agent_harness_service.tsindex 086781b917..1f3d8de064 100644--- a/infra/stacks/agent-harness-service/agent_harness_service.ts+++ b/infra/stacks/agent-harness-service/agent_harness_service.ts@@ -6,8 +6,7 @@ import {DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,ServiceTargetGroup,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,} from '../../packages/resources';import { EcrImage } from '../../packages/service';import {@@ -365,9 +364,7 @@ export class AgentHarnessService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,@@ -414,7 +411,7 @@ export class AgentHarnessService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: platform.family.toUpperCase(),cpuArchitecture:diff --git a/infra/stacks/agent-schedule-service/service.ts b/infra/stacks/agent-schedule-service/service.tsindex 66966868d6..6b77f427b3 100644--- a/infra/stacks/agent-schedule-service/service.ts+++ b/infra/stacks/agent-schedule-service/service.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -244,9 +243,7 @@ export class AgentScheduleService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,@@ -283,7 +280,7 @@ export class AgentScheduleService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: platform.family.toUpperCase(),cpuArchitecture:diff --git a/infra/stacks/authentication-service/service.ts b/infra/stacks/authentication-service/service.tsindex 1a42e0696b..81bb900d91 100644--- a/infra/stacks/authentication-service/service.ts+++ b/infra/stacks/authentication-service/service.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,serviceLoadBalancer,ServiceTargetGroup,} from '../../packages/resources';@@ -519,9 +518,7 @@ export class AuthenticationService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,@@ -568,7 +565,7 @@ export class AuthenticationService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/calendar-service/service.ts b/infra/stacks/calendar-service/service.tsindex a42ede282b..e2e75d811b 100644--- a/infra/stacks/calendar-service/service.ts+++ b/infra/stacks/calendar-service/service.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -232,9 +231,7 @@ export class CalendarService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: 'service',image: image.image.imageUri,@@ -275,7 +272,7 @@ export class CalendarService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/cloud-storage-service/cloud-storage-service.ts b/infra/stacks/cloud-storage-service/cloud-storage-service.tsindex 2765b15504..6ee9b06402 100644--- a/infra/stacks/cloud-storage-service/cloud-storage-service.ts+++ b/infra/stacks/cloud-storage-service/cloud-storage-service.ts@@ -7,8 +7,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -363,16 +362,14 @@ export class CloudStorageService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,stopTimeout: 10, // 10 seconds to force kill the taskcpu: stack === 'prod' ? 2048 : 512,memory: stack === 'prod' ? 4096 : 2048,- environment: containerEnvVars,+ environment: [...(containerEnvVars ?? [])],secrets: [...dopplerEcsEnvironment.containerSecrets],logConfiguration: {logDriver: 'awsfirelens',@@ -396,7 +393,7 @@ export class CloudStorageService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/cloud-storage-service/slack-import-worker.test.ts b/infra/stacks/cloud-storage-service/slack-import-worker.test.tsindex 6f3dcf2e00..c0fb761f10 100644--- a/infra/stacks/cloud-storage-service/slack-import-worker.test.ts+++ b/infra/stacks/cloud-storage-service/slack-import-worker.test.ts@@ -41,6 +41,8 @@ beforeAll(async () => {// Isolate the shared barrel's ESM cycle, keeping the real Doppler/image helpers.mock.module('../../packages/shared', () => ({stack: 'dev',+ grafanaTelemetryEnabled: true,+ grafanaIngestSecretArn: secretArn,config: new pulumi.Config('cloud-storage-service'),CLOUD_TRAIL_SNS_TOPIC_ARN: 'arn:aws:sns:us-east-1:123456789012:alerts',}));@@ -126,7 +128,7 @@ describe('Slack import Fargate worker', () => {});const task = service.taskDefinitionArgs;expect(task.cpu).toBe('1024');- expect(task.memory).toBe('2048');+ expect(task.memory).toBe('3072');expect(task.containers.worker.stopTimeout).toBe(120);expect(task.containers.worker.memory).toBe(1536);let memory = 0;@@ -141,7 +143,7 @@ describe('Slack import Fargate worker', () => {);expect(container.portMappings ?? []).toEqual([]);}- expect(memory).toBe(2048);+ expect(memory).toBe(2560);expect(service.deploymentCircuitBreaker).toEqual({enable: true,rollback: true,@@ -179,17 +181,9 @@ describe('Slack import Fargate worker', () => {);const task = resource('awsx:ecs:FargateService').inputs.taskDefinitionArgs;expect(task.taskRole.roleArn).not.toBe(task.executionRole.roleArn);- expect(task.containers.worker.secrets).toEqual(- [- 'DATABASE_URL',- 'INTERNAL_API_KEY',- 'SLACK_IMPORT_ENABLED',- 'SLACK_IMPORT_CONCURRENCY',- ].map((key) => ({- name: key,- valueFrom: `${secretArn}:${key}::`,- }))- );+ expect(task.containers.worker.secrets).toEqual([+ { name: 'APP_SECRETS_JSON', valueFrom: secretArn },+ ]);expect(resources.some((item) => item.name === 'slack-import-worker-execution-role'@@ -204,7 +198,7 @@ describe('Slack import Fargate worker', () => {).toBe(true);});- test('wires queue names, staging and the processing API rather than search queries', () => {+ test('keeps runtime telemetry identity alongside Doppler application settings', () => {const worker = resource('awsx:ecs:FargateService').inputs.taskDefinitionArgs.containers.worker;const env = Object.fromEntries(@@ -215,14 +209,9 @@ describe('Slack import Fargate worker', () => {);expect(env).toMatchObject({ENVIRONMENT: 'dev',- UPLOAD_STAGING_BUCKET: 'bulk-upload-staging-dev',- OVERRIDE_SLACK_IMPORT_QUEUE: 'slack-import-queue-dev',- OVERRIDE_SLACK_IMPORT_DLQ: 'slack-import-dlq-dev',- OVERRIDE_CONNECTION_GATEWAY_URL:- 'https://dev-gateway.macro.com/connection-gateway',- OVERRIDE_SEARCH_PROCESSING_SERVICE_URL:- 'https://dev-gateway.macro.com/search-processing',- SLACK_IMPORT_JOIN_EMAIL_ENABLED: 'false',+ DD_SERVICE: 'slack-import-worker',+ DD_ENV: 'dev',+ GRAFANA_OTLP_ENDPOINT: 'http://127.0.0.1:14317',});expect(env.SEARCH_SERVICE_URL).toBeUndefined();expect(env.OVERRIDE_SEARCH_SERVICE_URL).toBeUndefined();diff --git a/infra/stacks/cloud-storage-service/slack-import-worker.ts b/infra/stacks/cloud-storage-service/slack-import-worker.tsindex dbd7cdc880..92eeeec5f5 100644--- a/infra/stacks/cloud-storage-service/slack-import-worker.ts+++ b/infra/stacks/cloud-storage-service/slack-import-worker.ts@@ -1,15 +1,12 @@import * as aws from '@pulumi/aws';import * as awsx from '@pulumi/awsx';import * as pulumi from '@pulumi/pulumi';-import {- DATADOG_API_KEY,- datadogAgentContainer,- fargateLogRouterSidecarContainer,-} from '../../packages/resources/src/resources/datadog';+import { DATADOG_API_KEY } from '../../packages/resources/src/resources/datadog';import { DEFAULT_CONTINUE_BEFORE_STEADY_STATE } from '../../packages/resources/src/resources/ecs_deployment_defaults';import { EcsDeploymentFailureAlarm } from '../../packages/resources/src/resources/ecs_deployment_failure_alarm';+import { withTelemetry } from '../../packages/resources/src/resources/telemetry';import { EcrImage } from '../../packages/service/src/ecr';-import { config, stack } from '../../packages/shared';+import { config, grafanaTelemetryEnabled, stack } from '../../packages/shared';import { DopplerEcsEnvironment } from '../../packages/shared/src/doppler_environment';interface SlackImportWorkerArgs {@@ -118,47 +115,47 @@ export class SlackImportWorker extends pulumi.ComponentResource {},taskDefinitionArgs: {cpu: '1024',- memory: '2048',+ memory: grafanaTelemetryEnabled ? '3072' : '2048',taskRole: { roleArn: this.role.arn },executionRole: { roleArn: dopplerEcsEnvironment.executionRole.arn },runtimePlatform: {operatingSystemFamily: 'LINUX',cpuArchitecture: 'X86_64',},- containers: {- // Hard limits leave 1536 MiB for import work and 512 MiB for telemetry.- log_router: { ...fargateLogRouterSidecarContainer, memory: 128 },- datadog_agent: {- ...datadogAgentContainer,- memory: 384,- portMappings: [],- },- worker: {- name: SERVICE_NAME,- image: image.image.imageUri,- essential: true,- memory: 1536,- stopTimeout: 120,- secrets: [...dopplerEcsEnvironment.containerSecrets],- environment: [- { name: 'ENVIRONMENT', value: stack },- { name: 'DD_SERVICE', value: SERVICE_NAME },- { name: 'DD_ENV', value: stack },- ],- logConfiguration: {- logDriver: 'awsfirelens',- options: {- Name: 'datadog',- Host: 'http-intake.logs.us5.datadoghq.com',- apikey: DATADOG_API_KEY,- dd_service: SERVICE_NAME,- dd_source: 'fargate',- dd_tags: `env:${stack}`,- provider: 'ecs',+ containers: withTelemetry(+ SERVICE_NAME,+ {+ worker: {+ name: SERVICE_NAME,+ image: image.image.imageUri,+ essential: true,+ memory: 1536,+ stopTimeout: 120,+ secrets: [...dopplerEcsEnvironment.containerSecrets],+ environment: [+ { name: 'ENVIRONMENT', value: stack },+ { name: 'DD_SERVICE', value: SERVICE_NAME },+ { name: 'DD_ENV', value: stack },+ ],+ logConfiguration: {+ logDriver: 'awsfirelens',+ options: {+ Name: 'datadog',+ Host: 'http-intake.logs.us5.datadoghq.com',+ apikey: DATADOG_API_KEY,+ dd_service: SERVICE_NAME,+ dd_source: 'fargate',+ dd_tags: `env:${stack}`,+ provider: 'ecs',+ },},},},- },+ {+ logRouter: { memory: 128 },+ datadogAgent: { memory: 384, portMappings: [] },+ }+ ),},tags,},diff --git a/infra/stacks/connection-gateway/connection_gateway.ts b/infra/stacks/connection-gateway/connection_gateway.tsindex 0396bf2346..30471b0beb 100644--- a/infra/stacks/connection-gateway/connection_gateway.ts+++ b/infra/stacks/connection-gateway/connection_gateway.ts@@ -6,8 +6,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -204,16 +203,14 @@ export class ConnectionGateway extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,stopTimeout: 10, // 10 seconds to force kill the taskcpu: 4096,memory: 8192,- environment: containerEnvVars,+ environment: [...(containerEnvVars ?? [])],secrets: [...dopplerEcsEnvironment.containerSecrets],logConfiguration: {logDriver: 'awsfirelens',@@ -237,7 +234,7 @@ export class ConnectionGateway extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/contacts-service/service.ts b/infra/stacks/contacts-service/service.tsindex 226d0e1bfb..6b9484fd50 100644--- a/infra/stacks/contacts-service/service.ts+++ b/infra/stacks/contacts-service/service.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -214,9 +213,7 @@ export class ContactsService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,@@ -257,7 +254,7 @@ export class ContactsService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/convert-service/service.ts b/infra/stacks/convert-service/service.tsindex d7a75bab8b..cfd2b8135a 100644--- a/infra/stacks/convert-service/service.ts+++ b/infra/stacks/convert-service/service.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -210,9 +209,7 @@ export class ConvertService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,@@ -249,7 +246,7 @@ export class ConvertService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/document-cognition-service/document-cognition-service.ts b/infra/stacks/document-cognition-service/document-cognition-service.tsindex 6d04a5528c..3436247ad6 100644--- a/infra/stacks/document-cognition-service/document-cognition-service.ts+++ b/infra/stacks/document-cognition-service/document-cognition-service.ts@@ -6,8 +6,7 @@ import {DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,DatadogServiceEntity,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,serviceLoadBalancer,ServiceTargetGroup,} from '../../packages/resources';@@ -294,16 +293,14 @@ export class DocumentCognitionService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,stopTimeout: 10, // 10 seconds to force kill the taskcpu: 4096,memory: 8192,- environment: containerEnvVars,+ environment: [...(containerEnvVars ?? [])],secrets: [...dopplerEcsEnvironment.containerSecrets],logConfiguration: {logDriver: 'awsfirelens',@@ -327,7 +324,7 @@ export class DocumentCognitionService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/email-service/pubsub_workers.ts b/infra/stacks/email-service/pubsub_workers.tsindex 4d3daf902b..d4e75a9288 100644--- a/infra/stacks/email-service/pubsub_workers.ts+++ b/infra/stacks/email-service/pubsub_workers.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,} from '../../packages/resources';import { EcrImage } from '../../packages/service';import {@@ -107,9 +106,7 @@ export class EmailPubSubWorkers extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,@@ -132,7 +129,7 @@ export class EmailPubSubWorkers extends pulumi.ComponentResource {},// No portMappings - workers don't expose ports},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/email-service/service.ts b/infra/stacks/email-service/service.tsindex 703e2de8e0..76df1b507e 100644--- a/infra/stacks/email-service/service.ts+++ b/infra/stacks/email-service/service.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -144,9 +143,7 @@ export class EmailService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,@@ -177,7 +174,7 @@ export class EmailService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/image-proxy-service/image-proxy-service.ts b/infra/stacks/image-proxy-service/image-proxy-service.tsindex 370a37274f..745be69cdd 100644--- a/infra/stacks/image-proxy-service/image-proxy-service.ts+++ b/infra/stacks/image-proxy-service/image-proxy-service.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -194,16 +193,14 @@ export class ImageProxyService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,stopTimeout: 10,cpu: 512,memory: 1024,- environment: containerEnvVars,+ environment: [...(containerEnvVars ?? [])],secrets: [...dopplerEcsEnvironment.containerSecrets],logConfiguration: {logDriver: 'awsfirelens',@@ -227,7 +224,7 @@ export class ImageProxyService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/lexical-service/lexical-service.ts b/infra/stacks/lexical-service/lexical-service.tsindex 0ac502a390..53d523bf2d 100644--- a/infra/stacks/lexical-service/lexical-service.ts+++ b/infra/stacks/lexical-service/lexical-service.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -191,9 +190,7 @@ export class LexicalService extends pulumi.ComponentResource {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,@@ -202,7 +199,7 @@ export class LexicalService extends pulumi.ComponentResource {// conversions, and Bun runs a task's JS on one thread.cpu: 1024,memory: 2048,- environment: containerEnvVars,+ environment: [...(containerEnvVars ?? [])],secrets,logConfiguration: {logDriver: 'awsfirelens',@@ -226,7 +223,7 @@ export class LexicalService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/mcp-server/mcp-server.ts b/infra/stacks/mcp-server/mcp-server.tsindex d555d51462..319e44fb16 100644--- a/infra/stacks/mcp-server/mcp-server.ts+++ b/infra/stacks/mcp-server/mcp-server.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,serviceLoadBalancer,ServiceTargetGroup,} from '../../packages/resources';@@ -285,16 +284,14 @@ export class McpServer extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,stopTimeout: 10,cpu: 1024,memory: 2048,- environment: containerEnvVars,+ environment: [...(containerEnvVars ?? [])],secrets: [...dopplerEcsEnvironment.containerSecrets],logConfiguration: {logDriver: 'awsfirelens',@@ -318,7 +315,7 @@ export class McpServer extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/notification-service/service.ts b/infra/stacks/notification-service/service.tsindex aff3e7611b..f9519535e3 100644--- a/infra/stacks/notification-service/service.ts+++ b/infra/stacks/notification-service/service.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -270,16 +269,14 @@ export class NotificationService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,stopTimeout: 10, // 10 seconds to force kill the taskcpu: stack === 'prod' ? 1024 : 256,memory: stack === 'prod' ? 2048 : 512,- environment: containerEnvVars,+ environment: [...(containerEnvVars ?? [])],secrets: [...dopplerEcsEnvironment.containerSecrets],logConfiguration: {logDriver: 'awsfirelens',@@ -303,7 +300,7 @@ export class NotificationService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/observability/README.md b/infra/stacks/observability/README.mdindex 320248233e..2ebb350f5e 100644--- a/infra/stacks/observability/README.md+++ b/infra/stacks/observability/README.md@@ -324,6 +324,64 @@ crash. Metrics remote-write has a local WAL. EBS-backed backend WALs protect datalready delivered to Loki/Tempo. Establish failure/loss tolerance while Datadogstill receives the authoritative copy.+## Dev dual export++Every dev ECS service using the shared Datadog sidecars automatically also runs+Alloy; there is no per-service opt-in flag. Task definitions call the shared+`withTelemetry(serviceName, containers)` helper, which supplies the Datadog agent,+FireLens router, dev Alloy sidecar, and dev Grafana environment variable together.+Production keeps its existing Datadog configuration+until its rollout. The earlier `image-proxy-service` canary verified Ohio+ingestion; it used the superseded collector-in-front-of-Datadog design. The+current design keeps applications exporting directly to the Datadog agent. The execution role can read the ingestion-only+`observability/dev-ingest` secret in Virginia.+The application task role does not receive access to the Grafana OAuth bundle.++Applications keep their original Datadog trace exporter at loopback 4317. In dev+ECS tasks, `GRAFANA_OTLP_ENDPOINT` is injected automatically as+`http://127.0.0.1:14317` (registered in Doppler `observability/dev`). The shared+Rust entrypoint adds a second batch processor with its own bounded queue, copying+the same finished spans and trace IDs to Alloy. It does not change Datadog's+exporter, filtering, sampling or agent configuration. Local and production runs+do not add this exporter. This requires rebuilding the application images.++Alloy listens on loopback OTLP 14317/14318 and exports only to Ohio. FireLens+retains its existing Datadog output and parsing and adds a bounded-retry Fluent+Forward output to Alloy for logs. ECS metadata supplies task and container+metrics; identity is promoted into metric labels to keep replicas separate.+The extra container reserves 128 MiB with a 512 MiB hard limit; review Fargate+sizing during each stack preview. Alloy is non-essential: its failure does not+restart the application or Datadog. ECS does not automatically restart a stopped+non-essential container in this configuration; replace the task to recover it.++The Grafana branch batches asynchronously before its memory limiter. Overflow,+export failure, or task replacement can drop the optional copy. Failed optional+trace exports are deliberately suppressed so a Grafana outage does not generate+SDK export-error logs that feed existing Datadog alerts. The optional SDK queue+is limited to one 512-span batch, so draining it plus an in-flight batch fits the+five-second shutdown budget. Queue saturation can still emit SDK drop warnings. These queues are+not durable. Collector processes still share task resources, and FireLens shares+its input and parsing between the two outputs; this is not complete resource+isolation. Any future application metric SDK needs its own independent Grafana+reader/exporter; the Rust entrypoint currently exports traces, not app metrics.++The dev analytics proxy mirrors existing browser and worker OTLP traces/logs in+`waitUntil`, using a server-side token and a five-second export timeout. Browsers+send once. Payloads above 8 MiB skip only the mirror. Each Worker isolate admits+at most two concurrent copies; additional requests skip mirroring to bound+aggregate buffering. Datadog's response remains+the client response. `deploy-analytics-proxy-dev.yml` installs the token from+Secrets Manager and deploys only the dev Worker.++The manually dispatched `export-datadog-observability.yml` reads live dashboard/monitor definitions into a+seven-day GitHub artifact for the migration inventory. It never modifies Datadog. The current repository credentials return HTTP 403+for both reads; `monitors_read` and `dashboards_read` access must be restored+before claiming live inventory parity. Partial exports retain an explicit status file.+The live legacy agent-proxy/document-processing services outside the current+stack definitions and the CloudWatch-logged preview gateway need a separate+coverage pass. Lambda CloudWatch logs, RUM/session replay, synthetics and database query monitoring+are separate sources; ECS/edge OTLP duplication alone does not provide parity.+## Validation and subsequent passesFrom `infra/`, run `bunx biome check stacks/observability` and `bun run check`.diff --git a/infra/stacks/observability/migration/export-datadog.ts b/infra/stacks/observability/migration/export-datadog.tsnew file mode 100644index 0000000000..8ad3f142c1--- /dev/null+++ b/infra/stacks/observability/migration/export-datadog.ts@@ -0,0 +1,130 @@+import { mkdir, writeFile } from 'node:fs/promises';++const apiKey = process.env.DD_API_KEY;+const appKey = process.env.DD_APP_KEY;+if (!apiKey || !appKey)+ throw new Error('Datadog read credentials are required');+const headers = { 'DD-API-KEY': apiKey, 'DD-APPLICATION-KEY': appKey };+async function read(path: string): Promise<unknown> {+ const response = await fetch(`https://api.us5.datadoghq.com/api/v1/${path}`, {+ headers,+ signal: AbortSignal.timeout(30000),+ });+ if (!response.ok) throw new Error(`Datadog ${path}: HTTP ${response.status}`);+ return response.json();+}++const destination = process.argv[2];+if (!destination) throw new Error('Provide an output directory');+await mkdir(destination, { recursive: true });+function errorMessage(error: unknown): string {+ return error instanceof Error ? error.message : 'Export failed';+}+const exports = await Promise.allSettled([+ (async () => {+ const monitors: unknown[] = [];+ const errors: string[] = [];+ try {+ for (let page = 0; ; page++) {+ const batch = await read(`monitor?page=${page}&page_size=100`);+ if (!Array.isArray(batch)) throw new Error('Invalid monitor response');+ monitors.push(...batch);+ if (batch.length < 100) break;+ }+ } catch (error) {+ errors.push(errorMessage(error));+ }+ await writeFile(+ `${destination}/monitors.json`,+ JSON.stringify(monitors, null, 2)+ );+ return {+ kind: 'monitors',+ count: monitors.length,+ complete: errors.length === 0,+ errors,+ };+ })(),+ (async () => {+ const dashboards: { id: string }[] = [];+ const errors: string[] = [];+ try {+ for (let start = 0; ; start += 100) {+ const response = await read(`dashboard?count=100&start=${start}`);+ if (+ !response ||+ typeof response !== 'object' ||+ !('dashboards' in response) ||+ !Array.isArray(response.dashboards)+ )+ throw new Error('Invalid dashboard response');+ const entries: unknown[] = response.dashboards;+ for (const dashboard of entries) {+ if (+ !dashboard ||+ typeof dashboard !== 'object' ||+ !('id' in dashboard) ||+ typeof dashboard.id !== 'string' ||+ !/^[a-zA-Z0-9-]+$/.test(dashboard.id)+ )+ throw new Error('Invalid dashboard ID');+ try {+ const definition = await read(`dashboard/${dashboard.id}`);+ await writeFile(+ `${destination}/dashboard-${dashboard.id}.json`,+ JSON.stringify(definition, null, 2)+ );+ dashboards.push({ id: dashboard.id });+ } catch (error) {+ errors.push(`Dashboard ${dashboard.id}: ${errorMessage(error)}`);+ }+ }+ if (response.dashboards.length < 100) break;+ }+ } catch (error) {+ errors.push(errorMessage(error));+ }+ await writeFile(+ `${destination}/dashboards.json`,+ JSON.stringify(dashboards, null, 2)+ );+ return {+ kind: 'dashboards',+ count: dashboards.length,+ complete: errors.length === 0,+ errors,+ };+ })(),+]);+await writeFile(+ `${destination}/export-status.json`,+ JSON.stringify(+ exports.map((result) =>+ result.status === 'fulfilled'+ ? result.value+ : {+ error:+ result.reason instanceof Error+ ? result.reason.message+ : 'Export failed',+ }+ ),+ null,+ 2+ )+);+for (const result of exports) {+ if (result.status === 'fulfilled') {+ console.log(`Exported ${result.value.count} ${result.value.kind}`);+ for (const error of result.value.errors) console.error(error);+ } else+ console.error(+ result.reason instanceof Error ? result.reason.message : 'Export failed'+ );+}+if (+ exports.some(+ (result) => result.status === 'rejected' || !result.value.complete+ )+)+ process.exitCode = 1;diff --git a/infra/stacks/search-processing-service/service.ts b/infra/stacks/search-processing-service/service.tsindex 11184cc282..be6c36a553 100644--- a/infra/stacks/search-processing-service/service.ts+++ b/infra/stacks/search-processing-service/service.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -224,9 +223,7 @@ export class SearchProcessingService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,@@ -260,7 +257,7 @@ export class SearchProcessingService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/static-file-service/static-file-service.ts b/infra/stacks/static-file-service/static-file-service.tsindex d0ddd7f9a5..777cb07be4 100644--- a/infra/stacks/static-file-service/static-file-service.ts+++ b/infra/stacks/static-file-service/static-file-service.ts@@ -5,9 +5,8 @@ import {ALLOWED_ORIGINS,DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,- datadogAgentContainer,+ withTelemetry,EcsDeploymentFailureAlarm,- fargateLogRouterSidecarContainer,QueueAlarms,serviceLoadBalancer,} from '../../packages/resources';@@ -559,9 +558,7 @@ export class StaticFileService extends pulumi.ComponentResource {executionRole: {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: SERVICE_NAME,image: image.image.imageUri,@@ -598,7 +595,7 @@ export class StaticFileService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${args.platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/infra/stacks/unfurl-service/unfurl-service.ts b/infra/stacks/unfurl-service/unfurl-service.tsindex 4451060a60..15d6b6aa95 100644--- a/infra/stacks/unfurl-service/unfurl-service.ts+++ b/infra/stacks/unfurl-service/unfurl-service.ts@@ -5,8 +5,7 @@ import {DATADOG_API_KEY,DEFAULT_CONTINUE_BEFORE_STEADY_STATE,EcsDeploymentFailureAlarm,- datadogAgentContainer,- fargateLogRouterSidecarContainer,+ withTelemetry,ServiceTargetGroup,} from '../../packages/resources';import { EcrImage } from '../../packages/service';@@ -164,16 +163,14 @@ export class UnfurlService extends pulumi.ComponentResource {roleArn: dopplerEcsEnvironment.executionRole.arn,},- containers: {- log_router: fargateLogRouterSidecarContainer,- datadog_agent: datadogAgentContainer,+ containers: withTelemetry(BASE_NAME, {service: {name: BASE_NAME,image: image.image.imageUri,stopTimeout: 10, // 10 seconds to force kill the taskcpu: 256,memory: 512,- environment: containerEnvVars,+ environment: [...(containerEnvVars ?? [])],secrets: [...dopplerEcsEnvironment.containerSecrets],logConfiguration: {logDriver: 'awsfirelens',@@ -197,7 +194,7 @@ export class UnfurlService extends pulumi.ComponentResource {},],},- },+ }),runtimePlatform: {operatingSystemFamily: `${platform.family.toUpperCase()}`,cpuArchitecture: `${diff --git a/services/analytics-proxy/src/index.ts b/services/analytics-proxy/src/index.tsindex 1ea15a3408..6a14f189f6 100644--- a/services/analytics-proxy/src/index.ts+++ b/services/analytics-proxy/src/index.ts@@ -15,6 +15,9 @@ declare global {interface Env {/** Datadog API key for the OTLP traces intake. Unset in local dev. */DD_API_KEY?: string;+ /** Server-side copy enabled only in the dev proxy. */+ GRAFANA_OTLP_ENDPOINT?: string;+ GRAFANA_OTLP_TOKEN?: string;}}@@ -70,6 +73,73 @@ async function handleProxy(return await fetch(originRequest);}+// Each mirror can retain 8 MiB of chunks plus its assembled request body.+// Reserve before cloning so concurrent requests cannot accumulate extra copies.+const maximumConcurrentTelemetryCopies = 2;+let activeTelemetryCopies = 0;++async function copyTelemetry(+ request: Request,+ endpoint: string,+ path: string,+ token: string+): Promise<void> {+ if (activeTelemetryCopies >= maximumConcurrentTelemetryCopies) return;+ activeTelemetryCopies++;+ try {+ const copy = request.clone();+ const maximumBytes = 8 * 1024 * 1024;+ const reader = copy.body?.getReader();+ const chunks: Uint8Array[] = [];+ let length = 0;+ if (reader) {+ while (true) {+ const { done, value } = await reader.read();+ if (done) break;+ length += value.byteLength;+ if (length > maximumBytes) {+ await reader.cancel();+ console.warn('Grafana telemetry copy skipped: payload too large', {+ signal: path,+ });+ return;+ }+ chunks.push(value);+ }+ }+ const body = new Uint8Array(length);+ let offset = 0;+ for (const chunk of chunks) {+ body.set(chunk, offset);+ offset += chunk.byteLength;+ }+ // Keep browser credentials and the Datadog key out of the second request.+ const headers = new Headers({ Authorization: `Bearer ${token}` });+ for (const name of ['content-type', 'content-encoding']) {+ const value = request.headers.get(name);+ if (value) headers.set(name, value);+ }+ const response = await fetch(`${endpoint}${path}`, {+ method: 'POST',+ headers,+ body,+ redirect: 'error',+ signal: AbortSignal.timeout(5000),+ });+ if (!response.ok) {+ console.warn('Grafana telemetry copy rejected', {+ signal: path,+ status: response.status,+ });+ }+ await response.body?.cancel();+ } catch {+ console.warn('Grafana telemetry copy failed', { signal: path });+ } finally {+ activeTelemetryCopies--;+ }+}+const app = new Hono<{ Bindings: Env }>();// OTLP uses protobuf cross-origin. Cookies are stripped by handleProxy, so@@ -109,6 +179,19 @@ app.all(`${OTLP_PREFIX}/*`, async (c) => {const intake = otlpIntakeUrl(c.env, path);if (!intake) return c.text('Not found', 404);+ const mirrorEndpoint = c.env.GRAFANA_OTLP_ENDPOINT;+ const mirrorToken = c.env.GRAFANA_OTLP_TOKEN;+ if (+ c.req.method === 'POST' &&+ (path === '/v1/traces' || path === '/v1/logs') &&+ mirrorEndpoint &&+ mirrorToken+ ) {+ c.executionCtx.waitUntil(+ copyTelemetry(c.req.raw, mirrorEndpoint, path, mirrorToken)+ );+ }+return handleProxy(c.req.raw,intake,diff --git a/services/analytics-proxy/wrangler.jsonc b/services/analytics-proxy/wrangler.jsoncindex 6403cc324a..c8266b5e56 100644--- a/services/analytics-proxy/wrangler.jsonc+++ b/services/analytics-proxy/wrangler.jsonc@@ -51,6 +51,7 @@],"vars": {"POSTHOG_HOST": "https://us.i.posthog.com",+ "GRAFANA_OTLP_ENDPOINT": "https://otlp.macro-internal.com","OTLP_TRACES_INTAKE_URL": "https://cloudflare.integrations.otlp.us5.datadoghq.com","OTLP_LOGS_INTAKE_URL": "https://cloudflare.integrations.otlp.us5.datadoghq.com"}