Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Duplicate dev telemetry to Grafana while preserving direct Datadog export (#7863)

Wolf Mermelstein67abe0b87fcd
diff --git a/.github/workflows/deploy-analytics-proxy-dev.yml b/.github/workflows/deploy-analytics-proxy-dev.yml
new file mode 100644
index 0000000000..99ccb33cdc
--- /dev/null
+++ b/.github/workflows/deploy-analytics-proxy-dev.yml
@@ -0,0 +1,47 @@
+name: Deploy analytics proxy dev
+on:
+ push:
+ branches: [main]
+ paths:
+ - services/analytics-proxy/**
+ - .github/workflows/deploy-analytics-proxy-dev.yml
+ workflow_dispatch:
+permissions:
+ contents: read
+concurrency:
+ group: deploy-analytics-proxy-dev
+ cancel-in-progress: false
+jobs:
+ deploy:
+ runs-on: namespace-profile-linux-small
+ defaults:
+ run:
+ working-directory: services/analytics-proxy
+ steps:
+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10
+ with:
+ persist-credentials: false
+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
+ - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
+ with:
+ node-version: '22'
+ - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ aws-region: us-east-1
+ - run: bun install --frozen-lockfile
+ - run: bun run gen-types && bun run check
+ - name: Set server-side ingestion credential
+ env:
+ CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+ CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}
+ run: |
+ set -euo pipefail
+ aws secretsmanager get-secret-value --secret-id observability/dev-ingest --query SecretString --output text |
+ bunx wrangler secret put GRAFANA_OTLP_TOKEN --env dev
+ - name: Deploy dev proxy
+ run: bun run deploy-dev
+ env:
+ CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
+ CLOUDFLARE_ACCOUNT_ID: ${{ vars.CLOUDFLARE_ACCOUNT_ID }}
diff --git a/.github/workflows/export-datadog-observability.yml b/.github/workflows/export-datadog-observability.yml
new file mode 100644
index 0000000000..7e28452512
--- /dev/null
+++ b/.github/workflows/export-datadog-observability.yml
@@ -0,0 +1,24 @@
+name: Export Datadog observability definitions
+on:
+ workflow_dispatch:
+permissions:
+ contents: read
+jobs:
+ export:
+ runs-on: namespace-profile-linux-small
+ steps:
+ - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10
+ with:
+ persist-credentials: false
+ - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
+ - name: Read dashboards and monitors
+ env:
+ DD_API_KEY: ${{ secrets.DD_API_KEY }}
+ DD_APP_KEY: ${{ secrets.DD_APP_KEY }}
+ run: bun infra/stacks/observability/migration/export-datadog.ts /tmp/datadog-inventory
+ - if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+ with:
+ name: datadog-observability-definitions
+ path: /tmp/datadog-inventory
+ retention-days: 7
diff --git a/crates/macro_entrypoint/src/grafana.rs b/crates/macro_entrypoint/src/grafana.rs
new file mode 100644
index 0000000000..9830b11d4a
--- /dev/null
+++ b/crates/macro_entrypoint/src/grafana.rs
@@ -0,0 +1,35 @@
+//! Failure isolation for the optional Grafana trace copy.
+
+use opentelemetry_sdk::{
+ Resource,
+ error::OTelSdkResult,
+ trace::{SpanData, SpanExporter},
+};
+use std::time::Duration;
+
+/// Drops failed optional copies without producing SDK application-error logs in Datadog.
+#[derive(Debug)]
+pub(crate) struct GrafanaSpanExporter(pub(crate) opentelemetry_otlp::SpanExporter);
+
+impl SpanExporter for GrafanaSpanExporter {
+ async fn export(&self, batch: Vec<SpanData>) -> OTelSdkResult {
+ // The inner exporter has a two-second timeout. Returning its failure would
+ // make BatchSpanProcessor log ERROR through the application's Datadog sink.
+ let _ = self.0.export(batch).await;
+ Ok(())
+ }
+
+ fn set_resource(&mut self, resource: &Resource) {
+ self.0.set_resource(resource);
+ }
+
+ fn shutdown_with_timeout(&mut self, timeout: Duration) -> OTelSdkResult {
+ let _ = self.0.shutdown_with_timeout(timeout);
+ Ok(())
+ }
+
+ fn force_flush(&mut self) -> OTelSdkResult {
+ let _ = self.0.force_flush();
+ Ok(())
+ }
+}
diff --git a/crates/macro_entrypoint/src/lib.rs b/crates/macro_entrypoint/src/lib.rs
index 4d3a9de367..b83ba3ae13 100644
--- a/crates/macro_entrypoint/src/lib.rs
+++ b/crates/macro_entrypoint/src/lib.rs
@@ -3,6 +3,7 @@
//! This is used to provide consistent behaviour with e.g. tracing configurations
mod datadog_fmt;
+mod grafana;
mod shutdown;
#[cfg(test)]
@@ -36,6 +37,7 @@ maybe_env_vars! {
pub struct RustLog;
pub struct OtelExporterOtlpEndpoint;
pub struct OtelTraceFilter;
+ pub struct GrafanaOtlpEndpoint;
}
/// Build an [`EnvFilter`] from `RUST_LOG`, honoring values injected via `APP_SECRETS_JSON`.
@@ -167,7 +169,8 @@ impl MacroEntrypoint {
// LGTM stack), so local logs are queryable next to the traces
// they belong to.
let export_otel = OtelExporterOtlpEndpoint::new().is_some();
- let tracer_provider = export_otel.then(|| init_opentelemetry("local".to_string()));
+ let tracer_provider =
+ export_otel.then(|| init_opentelemetry("local".to_string(), None));
let logger_provider = export_otel.then(|| init_otel_logs("local".to_string()));
if let Some(provider) = tracer_provider.as_ref() {
@@ -241,7 +244,10 @@ impl MacroEntrypoint {
let env = DdEnv::new()
.map(|e| e.to_string())
.unwrap_or_else(|_| "unknown".to_string());
- let tracer_provider = init_opentelemetry(env);
+ let grafana_endpoint = matches!(self.env, Environment::Develop)
+ .then(GrafanaOtlpEndpoint::new)
+ .flatten();
+ let tracer_provider = init_opentelemetry(env, grafana_endpoint.as_deref());
let tracer = tracer_provider.tracer(service_name());
let rust_log_filter = rust_log_env_filter();
@@ -350,7 +356,10 @@ fn otel_resource(deployment_environment: String) -> opentelemetry_sdk::Resource
.build()
}
-fn init_opentelemetry(deployment_environment: String) -> SdkTracerProvider {
+fn init_opentelemetry(
+ deployment_environment: String,
+ grafana_endpoint: Option<&str>,
+) -> SdkTracerProvider {
// W3C trace-context propagation: lets macro_tower_layers parent request
// spans under an incoming `traceparent` (e.g. from the web app), and
// service clients propagate context onward.
@@ -364,10 +373,40 @@ fn init_opentelemetry(deployment_environment: String) -> SdkTracerProvider {
.build()
.expect("failed to create OTLP span exporter");
- SdkTracerProvider::builder()
+ let mut provider = SdkTracerProvider::builder()
.with_batch_exporter(exporter)
- .with_resource(otel_resource(deployment_environment))
- .build()
+ .with_resource(otel_resource(deployment_environment));
+
+ // Keep the original exporter and its queue independent. The second batch
+ // processor copies the same finished spans without changing IDs or sampling.
+ if let Some(endpoint) = grafana_endpoint.filter(|endpoint| !endpoint.trim().is_empty()) {
+ match opentelemetry_otlp::SpanExporter::builder()
+ .with_tonic()
+ .with_endpoint(endpoint)
+ .with_timeout(std::time::Duration::from_secs(2))
+ .build()
+ {
+ Ok(exporter) => {
+ // At most one in-flight and one queued batch: two two-second
+ // exports fit within the SDK's five-second shutdown timeout.
+ let processor = opentelemetry_sdk::trace::BatchSpanProcessor::builder(
+ grafana::GrafanaSpanExporter(exporter),
+ )
+ .with_batch_config(
+ opentelemetry_sdk::trace::BatchConfigBuilder::default()
+ .with_max_queue_size(512)
+ .with_max_export_batch_size(512)
+ .build(),
+ )
+ .build();
+ provider = provider.with_span_processor(processor);
+ }
+ // Tracing is not installed yet. Optional export must not prevent startup.
+ Err(error) => eprintln!("Grafana trace exporter unavailable: {error}"),
+ }
+ }
+
+ provider.build()
}
/// OTLP log export for the [`OpenTelemetryTracingBridge`]: tracing events
diff --git a/infra/packages/resources/src/index.ts b/infra/packages/resources/src/index.ts
index 68aeeef900..05ee70d021 100644
--- a/infra/packages/resources/src/index.ts
+++ b/infra/packages/resources/src/index.ts
@@ -37,3 +37,5 @@ export {
export type { TopicArgs } from './resources/kafka-cluster';
export { KafkaCluster } from './resources/kafka-cluster';
export { ServiceTargetGroup } from './resources/service_target_group';
+
+export { withTelemetry } from './resources/telemetry';
diff --git a/infra/packages/resources/src/resources/datadog.ts b/infra/packages/resources/src/resources/datadog.ts
index aa04713b28..c604176d08 100644
--- a/infra/packages/resources/src/resources/datadog.ts
+++ b/infra/packages/resources/src/resources/datadog.ts
@@ -5,7 +5,7 @@ import {
ComponentResource,
type ComponentResourceOptions,
} from '@pulumi/pulumi';
-import { stack } from '../../../shared';
+import { grafanaTelemetryEnabled, stack } from '../../../shared';
const DATADOG_API_KEY_SECRET_KEY = 'datadog-api-key';
@@ -78,7 +78,9 @@ export const fargateLogRouterSidecarContainer = {
type: 'fluentbit',
options: {
'config-file-type': 'file',
- 'config-file-value': '/fluent-bit/configs/parse-json.conf',
+ 'config-file-value': grafanaTelemetryEnabled
+ ? '/tmp/observability-extra.conf'
+ : '/fluent-bit/configs/parse-json.conf',
'enable-ecs-log-metadata': 'true',
},
},
@@ -96,6 +98,24 @@ export const fargateLogRouterSidecarContainer = {
value: stack,
},
],
+ ...(grafanaTelemetryEnabled
+ ? {
+ entryPoint: ['/bin/sh', '-ec'],
+ command: [
+ `cat /fluent-bit/configs/parse-json.conf > /tmp/observability-extra.conf
+cat >> /tmp/observability-extra.conf <<'CONFIG'
+[OUTPUT]
+ Name forward
+ Match *-firelens-*
+ Host 127.0.0.1
+ Port 24225
+ Retry_Limit 1
+ net.connect_timeout 2
+CONFIG
+exec /entrypoint.sh`,
+ ],
+ }
+ : {}),
memoryReservation: 50,
} satisfies ecs.TaskDefinitionContainerDefinitionArgs;
diff --git a/infra/packages/resources/src/resources/ecs-telemetry.alloy b/infra/packages/resources/src/resources/ecs-telemetry.alloy
new file mode 100644
index 0000000000..b0aa381429
--- /dev/null
+++ b/infra/packages/resources/src/resources/ecs-telemetry.alloy
@@ -0,0 +1,103 @@
+logging { level = "warn" }
+
+otelcol.receiver.otlp "applications" {
+ grpc { endpoint = "127.0.0.1:14317" }
+ http { endpoint = "127.0.0.1:14318" }
+ output {
+ traces = [otelcol.processor.batch.grafana.input]
+ metrics = [otelcol.processor.batch.grafana.input]
+ logs = [otelcol.processor.batch.grafana.input]
+ }
+}
+
+otelcol.receiver.fluentforward "application_logs" {
+ endpoint = "127.0.0.1:24225"
+ output { logs = [otelcol.processor.batch.grafana.input] }
+}
+
+otelcol.receiver.awsecscontainermetrics "task" {
+ collection_interval = "60s"
+ output { metrics = [otelcol.processor.batch.grafana.input] }
+}
+
+otelcol.processor.memory_limiter "grafana" {
+ check_interval = "1s"
+ limit = "384MiB"
+ spike_limit = "64MiB"
+ output {
+ logs = [otelcol.processor.transform.grafana.input]
+ traces = [otelcol.processor.transform.grafana.input]
+ metrics = [otelcol.processor.transform.grafana.input]
+ }
+}
+
+otelcol.processor.transform "grafana" {
+ error_mode = "ignore"
+ log_statements {
+ context = "log"
+ statements = [
+ `set(resource.attributes["service.name"], "` + sys.env("TELEMETRY_SERVICE") + `") where resource.attributes["service.name"] == nil`,
+ `set(resource.attributes["deployment.environment.name"], "dev")`,
+ `set(resource.attributes["deployment.environment"], "dev")`,
+ `set(attributes["container_name"], body["container_name"]) where IsMap(body)`,
+ `set(attributes["ecs_task_arn"], body["ecs_task_arn"]) where IsMap(body)`,
+ `set(body, body["log"]) where IsMap(body) and body["log"] != nil`,
+ ]
+ }
+ trace_statements {
+ context = "resource"
+ statements = [
+ `set(attributes["deployment.environment.name"], "dev")`,
+ `set(attributes["deployment.environment"], "dev")`,
+ ]
+ }
+ metric_statements {
+ context = "resource"
+ statements = [
+ `set(attributes["service.name"], "` + sys.env("TELEMETRY_SERVICE") + `") where attributes["service.name"] == nil`,
+ `set(attributes["deployment.environment.name"], "dev")`,
+ `set(attributes["deployment.environment"], "dev")`,
+ ]
+ }
+ // Preserve series identity when the destination converts OTLP to Prometheus.
+ metric_statements {
+ context = "datapoint"
+ statements = [
+ `set(attributes["deployment.environment"], "dev")`,
+ `set(attributes["aws.ecs.task.arn"], resource.attributes["aws.ecs.task.arn"]) where resource.attributes["aws.ecs.task.arn"] != nil`,
+ `set(attributes["container.id"], resource.attributes["container.id"]) where resource.attributes["container.id"] != nil`,
+ `set(attributes["container.name"], resource.attributes["container.name"]) where resource.attributes["container.name"] != nil`,
+ ]
+ }
+ output {
+ logs = [otelcol.exporter.otlphttp.grafana.input]
+ traces = [otelcol.exporter.otlphttp.grafana.input]
+ metrics = [otelcol.exporter.otlphttp.grafana.input]
+ }
+}
+
+// Buffer only the independent Grafana copy; Datadog bypasses this collector.
+otelcol.processor.batch "grafana" {
+ timeout = "5s"
+ send_batch_size = 256
+ send_batch_max_size = 512
+ output {
+ logs = [otelcol.processor.memory_limiter.grafana.input]
+ traces = [otelcol.processor.memory_limiter.grafana.input]
+ metrics = [otelcol.processor.memory_limiter.grafana.input]
+ }
+}
+
+otelcol.auth.bearer "grafana" {
+ token = sys.env("GRAFANA_OTLP_TOKEN")
+}
+
+otelcol.exporter.otlphttp "grafana" {
+ client {
+ endpoint = "https://otlp.macro-internal.com"
+ auth = otelcol.auth.bearer.grafana.handler
+ timeout = "5s"
+ }
+ sending_queue { queue_size = 128 }
+ retry_on_failure { max_elapsed_time = "60s" }
+}
diff --git a/infra/packages/resources/src/resources/grafana.ts b/infra/packages/resources/src/resources/grafana.ts
new file mode 100644
index 0000000000..9aabba8d13
--- /dev/null
+++ b/infra/packages/resources/src/resources/grafana.ts
@@ -0,0 +1,45 @@
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { ecs } from '@pulumi/awsx/types/input';
+import {
+ grafanaIngestSecretArn,
+ grafanaTelemetryEnabled,
+} from '../../../shared';
+
+// Applied automatically with the dev sidecar; never overrides the primary OTLP endpoint.
+export const grafanaTelemetryEnvironment = grafanaTelemetryEnabled
+ ? [{ name: 'GRAFANA_OTLP_ENDPOINT', value: 'http://127.0.0.1:14317' }]
+ : [];
+
+export function grafanaTelemetryContainers(
+ serviceName: string
+): Record<string, ecs.TaskDefinitionContainerDefinitionArgs> {
+ if (!grafanaTelemetryEnabled || !grafanaIngestSecretArn) return {};
+ return {
+ alloy: {
+ name: 'alloy',
+ image:
+ 'grafana/alloy:v1.16.0@sha256:6e00cf7c5a692ff5f24844529416ed017d76fce922f8199004e73d5eca46b6b8',
+ essential: false,
+ memory: 512,
+ memoryReservation: 128,
+ stopTimeout: 30,
+ user: '473',
+ entryPoint: ['/bin/sh', '-ec'],
+ command: [
+ 'umask 077; printf "%s" "$ALLOY_CONFIG" > /tmp/config.alloy; exec /bin/alloy run --stability.level=experimental --server.http.listen-addr=127.0.0.1:12345 --storage.path=/tmp/alloy /tmp/config.alloy',
+ ],
+ environment: [
+ {
+ name: 'ALLOY_CONFIG',
+ value: readFileSync(join(__dirname, 'ecs-telemetry.alloy'), 'utf8'),
+ },
+ { name: 'TELEMETRY_SERVICE', value: serviceName },
+ { name: 'GOMEMLIMIT', value: '400MiB' },
+ ],
+ secrets: [
+ { name: 'GRAFANA_OTLP_TOKEN', valueFrom: grafanaIngestSecretArn },
+ ],
+ },
+ };
+}
diff --git a/infra/packages/resources/src/resources/telemetry.ts b/infra/packages/resources/src/resources/telemetry.ts
new file mode 100644
index 0000000000..bcf52fb300
--- /dev/null
+++ b/infra/packages/resources/src/resources/telemetry.ts
@@ -0,0 +1,49 @@
+import type { ecs } from '@pulumi/awsx/types/input';
+import * as pulumi from '@pulumi/pulumi';
+import { grafanaTelemetryEnabled } from '../../../shared';
+import {
+ datadogAgentContainer,
+ fargateLogRouterSidecarContainer,
+} from './datadog';
+import {
+ grafanaTelemetryContainers,
+ grafanaTelemetryEnvironment,
+} from './grafana';
+
+/** Standard app container and telemetry sidecars, with Grafana copies in dev. */
+export function withTelemetry(
+ serviceName: string,
+ containers: Record<string, ecs.TaskDefinitionContainerDefinitionArgs>,
+ sidecars: {
+ logRouter?: Partial<ecs.TaskDefinitionContainerDefinitionArgs>;
+ datadogAgent?: Partial<ecs.TaskDefinitionContainerDefinitionArgs>;
+ } = {}
+): Record<string, ecs.TaskDefinitionContainerDefinitionArgs> {
+ return {
+ ...grafanaTelemetryContainers(serviceName),
+ log_router: {
+ ...fargateLogRouterSidecarContainer,
+ ...sidecars.logRouter,
+ },
+ datadog_agent: {
+ ...datadogAgentContainer,
+ ...sidecars.datadogAgent,
+ },
+ ...Object.fromEntries(
+ Object.entries(containers).map(([name, container]) => [
+ name,
+ grafanaTelemetryEnabled
+ ? {
+ ...container,
+ environment: pulumi
+ .output(container.environment)
+ .apply((environment) => [
+ ...(environment ?? []),
+ ...grafanaTelemetryEnvironment,
+ ]),
+ }
+ : container,
+ ])
+ ),
+ };
+}
diff --git a/infra/packages/shared/src/doppler_environment.ts b/infra/packages/shared/src/doppler_environment.ts
index 36df7f804b..9202a15c7f 100644
--- a/infra/packages/shared/src/doppler_environment.ts
+++ b/infra/packages/shared/src/doppler_environment.ts
@@ -1,6 +1,7 @@
import * as aws from '@pulumi/aws';
import * as pulumi from '@pulumi/pulumi';
import { stack } from '.';
+import { grafanaIngestSecretArn } from './grafana_telemetry';
export type DopplerEcsEnvironmentArgs = {
// Resource tags
@@ -35,7 +36,9 @@ export class DopplerEcsEnvironment extends pulumi.ComponentResource {
Statement: [
{
Action: ['secretsmanager:GetSecretValue'],
- Resource: pulumi.interpolate`${dopplerSecretSyncArn}`,
+ Resource: grafanaIngestSecretArn
+ ? [dopplerSecretSyncArn, grafanaIngestSecretArn]
+ : pulumi.interpolate`${dopplerSecretSyncArn}`,
Effect: 'Allow',
},
],
diff --git a/infra/packages/shared/src/grafana_telemetry.ts b/infra/packages/shared/src/grafana_telemetry.ts
new file mode 100644
index 0000000000..703db14b27
--- /dev/null
+++ b/infra/packages/shared/src/grafana_telemetry.ts
@@ -0,0 +1,10 @@
+import * as aws from '@pulumi/aws';
+import * as pulumi from '@pulumi/pulumi';
+
+// All dev ECS services run the shared telemetry sidecars. Production follows in a later rollout.
+export const grafanaTelemetryEnabled = pulumi.getStack() === 'dev';
+
+// Only the ingestion token is stored here; never grant tasks the Grafana OAuth bundle.
+export const grafanaIngestSecretArn = grafanaTelemetryEnabled
+ ? aws.secretsmanager.getSecretOutput({ name: 'observability/dev-ingest' }).arn
+ : undefined;
diff --git a/infra/packages/shared/src/index.ts b/infra/packages/shared/src/index.ts
index 3b2ea74b5f..26d85fa4ee 100644
--- a/infra/packages/shared/src/index.ts
+++ b/infra/packages/shared/src/index.ts
@@ -72,3 +72,8 @@ export { getGatewayAlb, type GatewayAlb } from './gateway';
export { GATEWAY_PRIORITIES, GatewayService } from './gateway_priorities';
export * from './service_urls';
+
+export {
+ grafanaTelemetryEnabled,
+ grafanaIngestSecretArn,
+} from './grafana_telemetry';
diff --git a/infra/stacks/agent-harness-service/agent_harness_service.ts b/infra/stacks/agent-harness-service/agent_harness_service.ts
index 086781b917..1f3d8de064 100644
--- a/infra/stacks/agent-harness-service/agent_harness_service.ts
+++ b/infra/stacks/agent-harness-service/agent_harness_service.ts
@@ -6,8 +6,7 @@ import {
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
ServiceTargetGroup,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
import {
@@ -365,9 +364,7 @@ export class AgentHarnessService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
@@ -414,7 +411,7 @@ export class AgentHarnessService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: platform.family.toUpperCase(),
cpuArchitecture:
diff --git a/infra/stacks/agent-schedule-service/service.ts b/infra/stacks/agent-schedule-service/service.ts
index 66966868d6..6b77f427b3 100644
--- a/infra/stacks/agent-schedule-service/service.ts
+++ b/infra/stacks/agent-schedule-service/service.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -244,9 +243,7 @@ export class AgentScheduleService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
@@ -283,7 +280,7 @@ export class AgentScheduleService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: platform.family.toUpperCase(),
cpuArchitecture:
diff --git a/infra/stacks/authentication-service/service.ts b/infra/stacks/authentication-service/service.ts
index 1a42e0696b..81bb900d91 100644
--- a/infra/stacks/authentication-service/service.ts
+++ b/infra/stacks/authentication-service/service.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
serviceLoadBalancer,
ServiceTargetGroup,
} from '../../packages/resources';
@@ -519,9 +518,7 @@ export class AuthenticationService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
@@ -568,7 +565,7 @@ export class AuthenticationService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/calendar-service/service.ts b/infra/stacks/calendar-service/service.ts
index a42ede282b..e2e75d811b 100644
--- a/infra/stacks/calendar-service/service.ts
+++ b/infra/stacks/calendar-service/service.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -232,9 +231,7 @@ export class CalendarService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: 'service',
image: image.image.imageUri,
@@ -275,7 +272,7 @@ export class CalendarService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/cloud-storage-service/cloud-storage-service.ts b/infra/stacks/cloud-storage-service/cloud-storage-service.ts
index 2765b15504..6ee9b06402 100644
--- a/infra/stacks/cloud-storage-service/cloud-storage-service.ts
+++ b/infra/stacks/cloud-storage-service/cloud-storage-service.ts
@@ -7,8 +7,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -363,16 +362,14 @@ export class CloudStorageService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
stopTimeout: 10, // 10 seconds to force kill the task
cpu: stack === 'prod' ? 2048 : 512,
memory: stack === 'prod' ? 4096 : 2048,
- environment: containerEnvVars,
+ environment: [...(containerEnvVars ?? [])],
secrets: [...dopplerEcsEnvironment.containerSecrets],
logConfiguration: {
logDriver: 'awsfirelens',
@@ -396,7 +393,7 @@ export class CloudStorageService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/cloud-storage-service/slack-import-worker.test.ts b/infra/stacks/cloud-storage-service/slack-import-worker.test.ts
index 6f3dcf2e00..c0fb761f10 100644
--- a/infra/stacks/cloud-storage-service/slack-import-worker.test.ts
+++ b/infra/stacks/cloud-storage-service/slack-import-worker.test.ts
@@ -41,6 +41,8 @@ beforeAll(async () => {
// Isolate the shared barrel's ESM cycle, keeping the real Doppler/image helpers.
mock.module('../../packages/shared', () => ({
stack: 'dev',
+ grafanaTelemetryEnabled: true,
+ grafanaIngestSecretArn: secretArn,
config: new pulumi.Config('cloud-storage-service'),
CLOUD_TRAIL_SNS_TOPIC_ARN: 'arn:aws:sns:us-east-1:123456789012:alerts',
}));
@@ -126,7 +128,7 @@ describe('Slack import Fargate worker', () => {
});
const task = service.taskDefinitionArgs;
expect(task.cpu).toBe('1024');
- expect(task.memory).toBe('2048');
+ expect(task.memory).toBe('3072');
expect(task.containers.worker.stopTimeout).toBe(120);
expect(task.containers.worker.memory).toBe(1536);
let memory = 0;
@@ -141,7 +143,7 @@ describe('Slack import Fargate worker', () => {
);
expect(container.portMappings ?? []).toEqual([]);
}
- expect(memory).toBe(2048);
+ expect(memory).toBe(2560);
expect(service.deploymentCircuitBreaker).toEqual({
enable: true,
rollback: true,
@@ -179,17 +181,9 @@ describe('Slack import Fargate worker', () => {
);
const task = resource('awsx:ecs:FargateService').inputs.taskDefinitionArgs;
expect(task.taskRole.roleArn).not.toBe(task.executionRole.roleArn);
- expect(task.containers.worker.secrets).toEqual(
- [
- 'DATABASE_URL',
- 'INTERNAL_API_KEY',
- 'SLACK_IMPORT_ENABLED',
- 'SLACK_IMPORT_CONCURRENCY',
- ].map((key) => ({
- name: key,
- valueFrom: `${secretArn}:${key}::`,
- }))
- );
+ expect(task.containers.worker.secrets).toEqual([
+ { name: 'APP_SECRETS_JSON', valueFrom: secretArn },
+ ]);
expect(
resources.some(
(item) => item.name === 'slack-import-worker-execution-role'
@@ -204,7 +198,7 @@ describe('Slack import Fargate worker', () => {
).toBe(true);
});
- test('wires queue names, staging and the processing API rather than search queries', () => {
+ test('keeps runtime telemetry identity alongside Doppler application settings', () => {
const worker = resource('awsx:ecs:FargateService').inputs.taskDefinitionArgs
.containers.worker;
const env = Object.fromEntries(
@@ -215,14 +209,9 @@ describe('Slack import Fargate worker', () => {
);
expect(env).toMatchObject({
ENVIRONMENT: 'dev',
- UPLOAD_STAGING_BUCKET: 'bulk-upload-staging-dev',
- OVERRIDE_SLACK_IMPORT_QUEUE: 'slack-import-queue-dev',
- OVERRIDE_SLACK_IMPORT_DLQ: 'slack-import-dlq-dev',
- OVERRIDE_CONNECTION_GATEWAY_URL:
- 'https://dev-gateway.macro.com/connection-gateway',
- OVERRIDE_SEARCH_PROCESSING_SERVICE_URL:
- 'https://dev-gateway.macro.com/search-processing',
- SLACK_IMPORT_JOIN_EMAIL_ENABLED: 'false',
+ DD_SERVICE: 'slack-import-worker',
+ DD_ENV: 'dev',
+ GRAFANA_OTLP_ENDPOINT: 'http://127.0.0.1:14317',
});
expect(env.SEARCH_SERVICE_URL).toBeUndefined();
expect(env.OVERRIDE_SEARCH_SERVICE_URL).toBeUndefined();
diff --git a/infra/stacks/cloud-storage-service/slack-import-worker.ts b/infra/stacks/cloud-storage-service/slack-import-worker.ts
index dbd7cdc880..92eeeec5f5 100644
--- a/infra/stacks/cloud-storage-service/slack-import-worker.ts
+++ b/infra/stacks/cloud-storage-service/slack-import-worker.ts
@@ -1,15 +1,12 @@
import * as aws from '@pulumi/aws';
import * as awsx from '@pulumi/awsx';
import * as pulumi from '@pulumi/pulumi';
-import {
- DATADOG_API_KEY,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
-} from '../../packages/resources/src/resources/datadog';
+import { DATADOG_API_KEY } from '../../packages/resources/src/resources/datadog';
import { DEFAULT_CONTINUE_BEFORE_STEADY_STATE } from '../../packages/resources/src/resources/ecs_deployment_defaults';
import { EcsDeploymentFailureAlarm } from '../../packages/resources/src/resources/ecs_deployment_failure_alarm';
+import { withTelemetry } from '../../packages/resources/src/resources/telemetry';
import { EcrImage } from '../../packages/service/src/ecr';
-import { config, stack } from '../../packages/shared';
+import { config, grafanaTelemetryEnabled, stack } from '../../packages/shared';
import { DopplerEcsEnvironment } from '../../packages/shared/src/doppler_environment';
interface SlackImportWorkerArgs {
@@ -118,47 +115,47 @@ export class SlackImportWorker extends pulumi.ComponentResource {
},
taskDefinitionArgs: {
cpu: '1024',
- memory: '2048',
+ memory: grafanaTelemetryEnabled ? '3072' : '2048',
taskRole: { roleArn: this.role.arn },
executionRole: { roleArn: dopplerEcsEnvironment.executionRole.arn },
runtimePlatform: {
operatingSystemFamily: 'LINUX',
cpuArchitecture: 'X86_64',
},
- containers: {
- // Hard limits leave 1536 MiB for import work and 512 MiB for telemetry.
- log_router: { ...fargateLogRouterSidecarContainer, memory: 128 },
- datadog_agent: {
- ...datadogAgentContainer,
- memory: 384,
- portMappings: [],
- },
- worker: {
- name: SERVICE_NAME,
- image: image.image.imageUri,
- essential: true,
- memory: 1536,
- stopTimeout: 120,
- secrets: [...dopplerEcsEnvironment.containerSecrets],
- environment: [
- { name: 'ENVIRONMENT', value: stack },
- { name: 'DD_SERVICE', value: SERVICE_NAME },
- { name: 'DD_ENV', value: stack },
- ],
- logConfiguration: {
- logDriver: 'awsfirelens',
- options: {
- Name: 'datadog',
- Host: 'http-intake.logs.us5.datadoghq.com',
- apikey: DATADOG_API_KEY,
- dd_service: SERVICE_NAME,
- dd_source: 'fargate',
- dd_tags: `env:${stack}`,
- provider: 'ecs',
+ containers: withTelemetry(
+ SERVICE_NAME,
+ {
+ worker: {
+ name: SERVICE_NAME,
+ image: image.image.imageUri,
+ essential: true,
+ memory: 1536,
+ stopTimeout: 120,
+ secrets: [...dopplerEcsEnvironment.containerSecrets],
+ environment: [
+ { name: 'ENVIRONMENT', value: stack },
+ { name: 'DD_SERVICE', value: SERVICE_NAME },
+ { name: 'DD_ENV', value: stack },
+ ],
+ logConfiguration: {
+ logDriver: 'awsfirelens',
+ options: {
+ Name: 'datadog',
+ Host: 'http-intake.logs.us5.datadoghq.com',
+ apikey: DATADOG_API_KEY,
+ dd_service: SERVICE_NAME,
+ dd_source: 'fargate',
+ dd_tags: `env:${stack}`,
+ provider: 'ecs',
+ },
},
},
},
- },
+ {
+ logRouter: { memory: 128 },
+ datadogAgent: { memory: 384, portMappings: [] },
+ }
+ ),
},
tags,
},
diff --git a/infra/stacks/connection-gateway/connection_gateway.ts b/infra/stacks/connection-gateway/connection_gateway.ts
index 0396bf2346..30471b0beb 100644
--- a/infra/stacks/connection-gateway/connection_gateway.ts
+++ b/infra/stacks/connection-gateway/connection_gateway.ts
@@ -6,8 +6,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -204,16 +203,14 @@ export class ConnectionGateway extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
stopTimeout: 10, // 10 seconds to force kill the task
cpu: 4096,
memory: 8192,
- environment: containerEnvVars,
+ environment: [...(containerEnvVars ?? [])],
secrets: [...dopplerEcsEnvironment.containerSecrets],
logConfiguration: {
logDriver: 'awsfirelens',
@@ -237,7 +234,7 @@ export class ConnectionGateway extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/contacts-service/service.ts b/infra/stacks/contacts-service/service.ts
index 226d0e1bfb..6b9484fd50 100644
--- a/infra/stacks/contacts-service/service.ts
+++ b/infra/stacks/contacts-service/service.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -214,9 +213,7 @@ export class ContactsService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
@@ -257,7 +254,7 @@ export class ContactsService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/convert-service/service.ts b/infra/stacks/convert-service/service.ts
index d7a75bab8b..cfd2b8135a 100644
--- a/infra/stacks/convert-service/service.ts
+++ b/infra/stacks/convert-service/service.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -210,9 +209,7 @@ export class ConvertService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
@@ -249,7 +246,7 @@ export class ConvertService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/document-cognition-service/document-cognition-service.ts b/infra/stacks/document-cognition-service/document-cognition-service.ts
index 6d04a5528c..3436247ad6 100644
--- a/infra/stacks/document-cognition-service/document-cognition-service.ts
+++ b/infra/stacks/document-cognition-service/document-cognition-service.ts
@@ -6,8 +6,7 @@ import {
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
DatadogServiceEntity,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
serviceLoadBalancer,
ServiceTargetGroup,
} from '../../packages/resources';
@@ -294,16 +293,14 @@ export class DocumentCognitionService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
stopTimeout: 10, // 10 seconds to force kill the task
cpu: 4096,
memory: 8192,
- environment: containerEnvVars,
+ environment: [...(containerEnvVars ?? [])],
secrets: [...dopplerEcsEnvironment.containerSecrets],
logConfiguration: {
logDriver: 'awsfirelens',
@@ -327,7 +324,7 @@ export class DocumentCognitionService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/email-service/pubsub_workers.ts b/infra/stacks/email-service/pubsub_workers.ts
index 4d3daf902b..d4e75a9288 100644
--- a/infra/stacks/email-service/pubsub_workers.ts
+++ b/infra/stacks/email-service/pubsub_workers.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
import {
@@ -107,9 +106,7 @@ export class EmailPubSubWorkers extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
@@ -132,7 +129,7 @@ export class EmailPubSubWorkers extends pulumi.ComponentResource {
},
// No portMappings - workers don't expose ports
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/email-service/service.ts b/infra/stacks/email-service/service.ts
index 703e2de8e0..76df1b507e 100644
--- a/infra/stacks/email-service/service.ts
+++ b/infra/stacks/email-service/service.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -144,9 +143,7 @@ export class EmailService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
@@ -177,7 +174,7 @@ export class EmailService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/image-proxy-service/image-proxy-service.ts b/infra/stacks/image-proxy-service/image-proxy-service.ts
index 370a37274f..745be69cdd 100644
--- a/infra/stacks/image-proxy-service/image-proxy-service.ts
+++ b/infra/stacks/image-proxy-service/image-proxy-service.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -194,16 +193,14 @@ export class ImageProxyService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
stopTimeout: 10,
cpu: 512,
memory: 1024,
- environment: containerEnvVars,
+ environment: [...(containerEnvVars ?? [])],
secrets: [...dopplerEcsEnvironment.containerSecrets],
logConfiguration: {
logDriver: 'awsfirelens',
@@ -227,7 +224,7 @@ export class ImageProxyService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/lexical-service/lexical-service.ts b/infra/stacks/lexical-service/lexical-service.ts
index 0ac502a390..53d523bf2d 100644
--- a/infra/stacks/lexical-service/lexical-service.ts
+++ b/infra/stacks/lexical-service/lexical-service.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -191,9 +190,7 @@ export class LexicalService extends pulumi.ComponentResource {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
@@ -202,7 +199,7 @@ export class LexicalService extends pulumi.ComponentResource {
// conversions, and Bun runs a task's JS on one thread.
cpu: 1024,
memory: 2048,
- environment: containerEnvVars,
+ environment: [...(containerEnvVars ?? [])],
secrets,
logConfiguration: {
logDriver: 'awsfirelens',
@@ -226,7 +223,7 @@ export class LexicalService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/mcp-server/mcp-server.ts b/infra/stacks/mcp-server/mcp-server.ts
index d555d51462..319e44fb16 100644
--- a/infra/stacks/mcp-server/mcp-server.ts
+++ b/infra/stacks/mcp-server/mcp-server.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
serviceLoadBalancer,
ServiceTargetGroup,
} from '../../packages/resources';
@@ -285,16 +284,14 @@ export class McpServer extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
stopTimeout: 10,
cpu: 1024,
memory: 2048,
- environment: containerEnvVars,
+ environment: [...(containerEnvVars ?? [])],
secrets: [...dopplerEcsEnvironment.containerSecrets],
logConfiguration: {
logDriver: 'awsfirelens',
@@ -318,7 +315,7 @@ export class McpServer extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/notification-service/service.ts b/infra/stacks/notification-service/service.ts
index aff3e7611b..f9519535e3 100644
--- a/infra/stacks/notification-service/service.ts
+++ b/infra/stacks/notification-service/service.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -270,16 +269,14 @@ export class NotificationService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
stopTimeout: 10, // 10 seconds to force kill the task
cpu: stack === 'prod' ? 1024 : 256,
memory: stack === 'prod' ? 2048 : 512,
- environment: containerEnvVars,
+ environment: [...(containerEnvVars ?? [])],
secrets: [...dopplerEcsEnvironment.containerSecrets],
logConfiguration: {
logDriver: 'awsfirelens',
@@ -303,7 +300,7 @@ export class NotificationService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/observability/README.md b/infra/stacks/observability/README.md
index 320248233e..2ebb350f5e 100644
--- a/infra/stacks/observability/README.md
+++ b/infra/stacks/observability/README.md
@@ -324,6 +324,64 @@ crash. Metrics remote-write has a local WAL. EBS-backed backend WALs protect dat
already delivered to Loki/Tempo. Establish failure/loss tolerance while Datadog
still receives the authoritative copy.
+## Dev dual export
+
+Every dev ECS service using the shared Datadog sidecars automatically also runs
+Alloy; there is no per-service opt-in flag. Task definitions call the shared
+`withTelemetry(serviceName, containers)` helper, which supplies the Datadog agent,
+FireLens router, dev Alloy sidecar, and dev Grafana environment variable together.
+Production keeps its existing Datadog configuration
+until its rollout. The earlier `image-proxy-service` canary verified Ohio
+ingestion; it used the superseded collector-in-front-of-Datadog design. The
+current design keeps applications exporting directly to the Datadog agent. The execution role can read the ingestion-only
+`observability/dev-ingest` secret in Virginia.
+The application task role does not receive access to the Grafana OAuth bundle.
+
+Applications keep their original Datadog trace exporter at loopback 4317. In dev
+ECS tasks, `GRAFANA_OTLP_ENDPOINT` is injected automatically as
+`http://127.0.0.1:14317` (registered in Doppler `observability/dev`). The shared
+Rust entrypoint adds a second batch processor with its own bounded queue, copying
+the same finished spans and trace IDs to Alloy. It does not change Datadog's
+exporter, filtering, sampling or agent configuration. Local and production runs
+do not add this exporter. This requires rebuilding the application images.
+
+Alloy listens on loopback OTLP 14317/14318 and exports only to Ohio. FireLens
+retains its existing Datadog output and parsing and adds a bounded-retry Fluent
+Forward output to Alloy for logs. ECS metadata supplies task and container
+metrics; identity is promoted into metric labels to keep replicas separate.
+The extra container reserves 128 MiB with a 512 MiB hard limit; review Fargate
+sizing during each stack preview. Alloy is non-essential: its failure does not
+restart the application or Datadog. ECS does not automatically restart a stopped
+non-essential container in this configuration; replace the task to recover it.
+
+The Grafana branch batches asynchronously before its memory limiter. Overflow,
+export failure, or task replacement can drop the optional copy. Failed optional
+trace exports are deliberately suppressed so a Grafana outage does not generate
+SDK export-error logs that feed existing Datadog alerts. The optional SDK queue
+is limited to one 512-span batch, so draining it plus an in-flight batch fits the
+five-second shutdown budget. Queue saturation can still emit SDK drop warnings. These queues are
+not durable. Collector processes still share task resources, and FireLens shares
+its input and parsing between the two outputs; this is not complete resource
+isolation. Any future application metric SDK needs its own independent Grafana
+reader/exporter; the Rust entrypoint currently exports traces, not app metrics.
+
+The dev analytics proxy mirrors existing browser and worker OTLP traces/logs in
+`waitUntil`, using a server-side token and a five-second export timeout. Browsers
+send once. Payloads above 8 MiB skip only the mirror. Each Worker isolate admits
+at most two concurrent copies; additional requests skip mirroring to bound
+aggregate buffering. Datadog's response remains
+the client response. `deploy-analytics-proxy-dev.yml` installs the token from
+Secrets Manager and deploys only the dev Worker.
+
+The manually dispatched `export-datadog-observability.yml` reads live dashboard/monitor definitions into a
+seven-day GitHub artifact for the migration inventory. It never modifies Datadog. The current repository credentials return HTTP 403
+for both reads; `monitors_read` and `dashboards_read` access must be restored
+before claiming live inventory parity. Partial exports retain an explicit status file.
+The live legacy agent-proxy/document-processing services outside the current
+stack definitions and the CloudWatch-logged preview gateway need a separate
+coverage pass. Lambda CloudWatch logs, RUM/session replay, synthetics and database query monitoring
+are separate sources; ECS/edge OTLP duplication alone does not provide parity.
+
## Validation and subsequent passes
From `infra/`, run `bunx biome check stacks/observability` and `bun run check`.
diff --git a/infra/stacks/observability/migration/export-datadog.ts b/infra/stacks/observability/migration/export-datadog.ts
new file mode 100644
index 0000000000..8ad3f142c1
--- /dev/null
+++ b/infra/stacks/observability/migration/export-datadog.ts
@@ -0,0 +1,130 @@
+import { mkdir, writeFile } from 'node:fs/promises';
+
+const apiKey = process.env.DD_API_KEY;
+const appKey = process.env.DD_APP_KEY;
+if (!apiKey || !appKey)
+ throw new Error('Datadog read credentials are required');
+const headers = { 'DD-API-KEY': apiKey, 'DD-APPLICATION-KEY': appKey };
+async function read(path: string): Promise<unknown> {
+ const response = await fetch(`https://api.us5.datadoghq.com/api/v1/${path}`, {
+ headers,
+ signal: AbortSignal.timeout(30000),
+ });
+ if (!response.ok) throw new Error(`Datadog ${path}: HTTP ${response.status}`);
+ return response.json();
+}
+
+const destination = process.argv[2];
+if (!destination) throw new Error('Provide an output directory');
+await mkdir(destination, { recursive: true });
+function errorMessage(error: unknown): string {
+ return error instanceof Error ? error.message : 'Export failed';
+}
+const exports = await Promise.allSettled([
+ (async () => {
+ const monitors: unknown[] = [];
+ const errors: string[] = [];
+ try {
+ for (let page = 0; ; page++) {
+ const batch = await read(`monitor?page=${page}&page_size=100`);
+ if (!Array.isArray(batch)) throw new Error('Invalid monitor response');
+ monitors.push(...batch);
+ if (batch.length < 100) break;
+ }
+ } catch (error) {
+ errors.push(errorMessage(error));
+ }
+ await writeFile(
+ `${destination}/monitors.json`,
+ JSON.stringify(monitors, null, 2)
+ );
+ return {
+ kind: 'monitors',
+ count: monitors.length,
+ complete: errors.length === 0,
+ errors,
+ };
+ })(),
+ (async () => {
+ const dashboards: { id: string }[] = [];
+ const errors: string[] = [];
+ try {
+ for (let start = 0; ; start += 100) {
+ const response = await read(`dashboard?count=100&start=${start}`);
+ if (
+ !response ||
+ typeof response !== 'object' ||
+ !('dashboards' in response) ||
+ !Array.isArray(response.dashboards)
+ )
+ throw new Error('Invalid dashboard response');
+ const entries: unknown[] = response.dashboards;
+ for (const dashboard of entries) {
+ if (
+ !dashboard ||
+ typeof dashboard !== 'object' ||
+ !('id' in dashboard) ||
+ typeof dashboard.id !== 'string' ||
+ !/^[a-zA-Z0-9-]+$/.test(dashboard.id)
+ )
+ throw new Error('Invalid dashboard ID');
+ try {
+ const definition = await read(`dashboard/${dashboard.id}`);
+ await writeFile(
+ `${destination}/dashboard-${dashboard.id}.json`,
+ JSON.stringify(definition, null, 2)
+ );
+ dashboards.push({ id: dashboard.id });
+ } catch (error) {
+ errors.push(`Dashboard ${dashboard.id}: ${errorMessage(error)}`);
+ }
+ }
+ if (response.dashboards.length < 100) break;
+ }
+ } catch (error) {
+ errors.push(errorMessage(error));
+ }
+ await writeFile(
+ `${destination}/dashboards.json`,
+ JSON.stringify(dashboards, null, 2)
+ );
+ return {
+ kind: 'dashboards',
+ count: dashboards.length,
+ complete: errors.length === 0,
+ errors,
+ };
+ })(),
+]);
+await writeFile(
+ `${destination}/export-status.json`,
+ JSON.stringify(
+ exports.map((result) =>
+ result.status === 'fulfilled'
+ ? result.value
+ : {
+ error:
+ result.reason instanceof Error
+ ? result.reason.message
+ : 'Export failed',
+ }
+ ),
+ null,
+ 2
+ )
+);
+for (const result of exports) {
+ if (result.status === 'fulfilled') {
+ console.log(`Exported ${result.value.count} ${result.value.kind}`);
+ for (const error of result.value.errors) console.error(error);
+ } else
+ console.error(
+ result.reason instanceof Error ? result.reason.message : 'Export failed'
+ );
+}
+if (
+ exports.some(
+ (result) => result.status === 'rejected' || !result.value.complete
+ )
+)
+ process.exitCode = 1;
diff --git a/infra/stacks/search-processing-service/service.ts b/infra/stacks/search-processing-service/service.ts
index 11184cc282..be6c36a553 100644
--- a/infra/stacks/search-processing-service/service.ts
+++ b/infra/stacks/search-processing-service/service.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -224,9 +223,7 @@ export class SearchProcessingService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
@@ -260,7 +257,7 @@ export class SearchProcessingService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/static-file-service/static-file-service.ts b/infra/stacks/static-file-service/static-file-service.ts
index d0ddd7f9a5..777cb07be4 100644
--- a/infra/stacks/static-file-service/static-file-service.ts
+++ b/infra/stacks/static-file-service/static-file-service.ts
@@ -5,9 +5,8 @@ import {
ALLOWED_ORIGINS,
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
- datadogAgentContainer,
+ withTelemetry,
EcsDeploymentFailureAlarm,
- fargateLogRouterSidecarContainer,
QueueAlarms,
serviceLoadBalancer,
} from '../../packages/resources';
@@ -559,9 +558,7 @@ export class StaticFileService extends pulumi.ComponentResource {
executionRole: {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: SERVICE_NAME,
image: image.image.imageUri,
@@ -598,7 +595,7 @@ export class StaticFileService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${args.platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/infra/stacks/unfurl-service/unfurl-service.ts b/infra/stacks/unfurl-service/unfurl-service.ts
index 4451060a60..15d6b6aa95 100644
--- a/infra/stacks/unfurl-service/unfurl-service.ts
+++ b/infra/stacks/unfurl-service/unfurl-service.ts
@@ -5,8 +5,7 @@ import {
DATADOG_API_KEY,
DEFAULT_CONTINUE_BEFORE_STEADY_STATE,
EcsDeploymentFailureAlarm,
- datadogAgentContainer,
- fargateLogRouterSidecarContainer,
+ withTelemetry,
ServiceTargetGroup,
} from '../../packages/resources';
import { EcrImage } from '../../packages/service';
@@ -164,16 +163,14 @@ export class UnfurlService extends pulumi.ComponentResource {
roleArn: dopplerEcsEnvironment.executionRole.arn,
},
- containers: {
- log_router: fargateLogRouterSidecarContainer,
- datadog_agent: datadogAgentContainer,
+ containers: withTelemetry(BASE_NAME, {
service: {
name: BASE_NAME,
image: image.image.imageUri,
stopTimeout: 10, // 10 seconds to force kill the task
cpu: 256,
memory: 512,
- environment: containerEnvVars,
+ environment: [...(containerEnvVars ?? [])],
secrets: [...dopplerEcsEnvironment.containerSecrets],
logConfiguration: {
logDriver: 'awsfirelens',
@@ -197,7 +194,7 @@ export class UnfurlService extends pulumi.ComponentResource {
},
],
},
- },
+ }),
runtimePlatform: {
operatingSystemFamily: `${platform.family.toUpperCase()}`,
cpuArchitecture: `${
diff --git a/services/analytics-proxy/src/index.ts b/services/analytics-proxy/src/index.ts
index 1ea15a3408..6a14f189f6 100644
--- a/services/analytics-proxy/src/index.ts
+++ b/services/analytics-proxy/src/index.ts
@@ -15,6 +15,9 @@ declare global {
interface Env {
/** Datadog API key for the OTLP traces intake. Unset in local dev. */
DD_API_KEY?: string;
+ /** Server-side copy enabled only in the dev proxy. */
+ GRAFANA_OTLP_ENDPOINT?: string;
+ GRAFANA_OTLP_TOKEN?: string;
}
}
@@ -70,6 +73,73 @@ async function handleProxy(
return await fetch(originRequest);
}
+// Each mirror can retain 8 MiB of chunks plus its assembled request body.
+// Reserve before cloning so concurrent requests cannot accumulate extra copies.
+const maximumConcurrentTelemetryCopies = 2;
+let activeTelemetryCopies = 0;
+
+async function copyTelemetry(
+ request: Request,
+ endpoint: string,
+ path: string,
+ token: string
+): Promise<void> {
+ if (activeTelemetryCopies >= maximumConcurrentTelemetryCopies) return;
+ activeTelemetryCopies++;
+ try {
+ const copy = request.clone();
+ const maximumBytes = 8 * 1024 * 1024;
+ const reader = copy.body?.getReader();
+ const chunks: Uint8Array[] = [];
+ let length = 0;
+ if (reader) {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ length += value.byteLength;
+ if (length > maximumBytes) {
+ await reader.cancel();
+ console.warn('Grafana telemetry copy skipped: payload too large', {
+ signal: path,
+ });
+ return;
+ }
+ chunks.push(value);
+ }
+ }
+ const body = new Uint8Array(length);
+ let offset = 0;
+ for (const chunk of chunks) {
+ body.set(chunk, offset);
+ offset += chunk.byteLength;
+ }
+ // Keep browser credentials and the Datadog key out of the second request.
+ const headers = new Headers({ Authorization: `Bearer ${token}` });
+ for (const name of ['content-type', 'content-encoding']) {
+ const value = request.headers.get(name);
+ if (value) headers.set(name, value);
+ }
+ const response = await fetch(`${endpoint}${path}`, {
+ method: 'POST',
+ headers,
+ body,
+ redirect: 'error',
+ signal: AbortSignal.timeout(5000),
+ });
+ if (!response.ok) {
+ console.warn('Grafana telemetry copy rejected', {
+ signal: path,
+ status: response.status,
+ });
+ }
+ await response.body?.cancel();
+ } catch {
+ console.warn('Grafana telemetry copy failed', { signal: path });
+ } finally {
+ activeTelemetryCopies--;
+ }
+}
+
const app = new Hono<{ Bindings: Env }>();
// OTLP uses protobuf cross-origin. Cookies are stripped by handleProxy, so
@@ -109,6 +179,19 @@ app.all(`${OTLP_PREFIX}/*`, async (c) => {
const intake = otlpIntakeUrl(c.env, path);
if (!intake) return c.text('Not found', 404);
+ const mirrorEndpoint = c.env.GRAFANA_OTLP_ENDPOINT;
+ const mirrorToken = c.env.GRAFANA_OTLP_TOKEN;
+ if (
+ c.req.method === 'POST' &&
+ (path === '/v1/traces' || path === '/v1/logs') &&
+ mirrorEndpoint &&
+ mirrorToken
+ ) {
+ c.executionCtx.waitUntil(
+ copyTelemetry(c.req.raw, mirrorEndpoint, path, mirrorToken)
+ );
+ }
+
return handleProxy(
c.req.raw,
intake,
diff --git a/services/analytics-proxy/wrangler.jsonc b/services/analytics-proxy/wrangler.jsonc
index 6403cc324a..c8266b5e56 100644
--- a/services/analytics-proxy/wrangler.jsonc
+++ b/services/analytics-proxy/wrangler.jsonc
@@ -51,6 +51,7 @@
],
"vars": {
"POSTHOG_HOST": "https://us.i.posthog.com",
+ "GRAFANA_OTLP_ENDPOINT": "https://otlp.macro-internal.com",
"OTLP_TRACES_INTAKE_URL": "https://cloudflare.integrations.otlp.us5.datadoghq.com",
"OTLP_LOGS_INTAKE_URL": "https://cloudflare.integrations.otlp.us5.datadoghq.com"
}