Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(infra): declare every Datadog monitor in code (#6065)

Wolf Mermelstein60284505495e
diff --git a/crates/documents/src/outbound/editing_worker_client.rs b/crates/documents/src/outbound/editing_worker_client.rs
index 330178feeb..3f1d5d4fb4 100644
--- a/crates/documents/src/outbound/editing_worker_client.rs
+++ b/crates/documents/src/outbound/editing_worker_client.rs
@@ -3,10 +3,14 @@
use crate::domain::ports::editing::{
EditMode, EditResult, EditUsage, EditingWorkerService, EditorName,
};
+use anyhow::Context;
use macro_sync_service_jwt::DocumentPermissionToken;
use reqwest::Client;
use std::sync::Arc;
+#[cfg(test)]
+mod test;
+
/// Reqwest-backed client for the AI editing worker.
#[derive(Clone)]
pub struct ReqwestEditingWorkerClient {
@@ -230,7 +234,8 @@ impl EditingWorkerService for ReqwestEditingWorkerClient {
.headers(headers)
.json(&request_body)
.send()
- .await?;
+ .await
+ .context("editing worker request failed")?;
let status = edit_resp.status();
if !status.is_success() {
diff --git a/crates/documents/src/outbound/editing_worker_client/test.rs b/crates/documents/src/outbound/editing_worker_client/test.rs
new file mode 100644
index 0000000000..ff6c38961a
--- /dev/null
+++ b/crates/documents/src/outbound/editing_worker_client/test.rs
@@ -0,0 +1,65 @@
+use super::*;
+use std::time::Duration;
+
+#[tokio::test]
+async fn connection_failure_has_monitorable_context_and_preserves_cause() {
+ // Reserve a port without listening so connection attempts are refused.
+ let socket = tokio::net::TcpSocket::new_v4().unwrap();
+ socket.bind("127.0.0.1:0".parse().unwrap()).unwrap();
+ let client = ReqwestEditingWorkerClient::new(
+ format!("http://{}", socket.local_addr().unwrap()),
+ Arc::new(
+ Client::builder()
+ .no_proxy()
+ .timeout(Duration::from_secs(5))
+ .build()
+ .unwrap(),
+ ),
+ );
+
+ let error = client
+ .edit(
+ "test-document",
+ &"test-token".to_owned().into(),
+ "edit",
+ EditMode::Supervised,
+ None,
+ )
+ .await
+ .err()
+ .expect("a connection to a port that is not listening must fail");
+
+ assert_eq!(error.to_string(), "editing worker request failed");
+ assert!(error.downcast_ref::<reqwest::Error>().unwrap().is_connect());
+}
+
+#[tokio::test]
+async fn timeout_has_monitorable_context_and_preserves_cause() {
+ // Keep the connection queued without serving a response.
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let client = ReqwestEditingWorkerClient::new(
+ format!("http://{}", listener.local_addr().unwrap()),
+ Arc::new(
+ Client::builder()
+ .no_proxy()
+ .timeout(Duration::from_millis(100))
+ .build()
+ .unwrap(),
+ ),
+ );
+
+ let error = client
+ .edit(
+ "test-document",
+ &"test-token".to_owned().into(),
+ "edit",
+ EditMode::Supervised,
+ None,
+ )
+ .await
+ .err()
+ .expect("a worker that never responds must time out");
+
+ assert_eq!(error.to_string(), "editing worker request failed");
+ assert!(error.downcast_ref::<reqwest::Error>().unwrap().is_timeout());
+}
diff --git a/infra/bun.lock b/infra/bun.lock
index a852f1f779..2c2c1beb34 100644
--- a/infra/bun.lock
+++ b/infra/bun.lock
@@ -171,6 +171,14 @@
"name": "mcp-server-stack",
"version": "0.0.0",
},
+ "stacks/monitoring": {
+ "name": "monitoring-stack",
+ "version": "0.0.0",
+ "dependencies": {
+ "@pulumi/datadog": "^4.51.0",
+ "@pulumi/pulumi": "^3.181.0",
+ },
+ },
"stacks/notification-service": {
"name": "notification-service-stack",
"version": "0.0.0",
@@ -695,6 +703,8 @@
"module-details-from-path": ["[email protected]", "", {}, "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w=="],
+ "monitoring-stack": ["monitoring-stack@workspace:stacks/monitoring"],
+
"ms": ["[email protected]", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
"negotiator": ["[email protected]", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="],
diff --git a/infra/stacks/monitoring/.gitignore b/infra/stacks/monitoring/.gitignore
new file mode 100644
index 0000000000..9d5bad2c75
--- /dev/null
+++ b/infra/stacks/monitoring/.gitignore
@@ -0,0 +1,2 @@
+# Emitted by `pulumi import --out` during adoption; never committed.
+imported.generated.ts
diff --git a/infra/stacks/monitoring/Pulumi.prod.yaml b/infra/stacks/monitoring/Pulumi.prod.yaml
new file mode 100644
index 0000000000..138f62896b
--- /dev/null
+++ b/infra/stacks/monitoring/Pulumi.prod.yaml
@@ -0,0 +1,4 @@
+# Datadog monitors are org-global, so this project uses only the prod stack.
+# The provider reads credentials and the Datadog site from DD_API_KEY,
+# DD_APP_KEY, and DD_HOST, supplied locally or by the deployment workflow.
+config: {}
diff --git a/infra/stacks/monitoring/Pulumi.yaml b/infra/stacks/monitoring/Pulumi.yaml
new file mode 100644
index 0000000000..e421141759
--- /dev/null
+++ b/infra/stacks/monitoring/Pulumi.yaml
@@ -0,0 +1,3 @@
+name: monitoring
+runtime: nodejs
+description: Datadog monitors and synthetic tests
diff --git a/infra/stacks/monitoring/README.md b/infra/stacks/monitoring/README.md
new file mode 100644
index 0000000000..852793d3a6
--- /dev/null
+++ b/infra/stacks/monitoring/README.md
@@ -0,0 +1,107 @@
+# monitoring
+
+Datadog monitors and synthetic tests as code.
+
+36 resources here (35 monitors and one synthetic test) already exist in
+Datadog. This stack **adopts** them — it does not recreate them. `import.json`
+lists every one by id. The two AI-editing monitors in `monitors/ai-editing.ts`
+are new and are meant to be created.
+
+## Read this before deploying
+
+**Run the import before the first `pulumi up`.** The program declares 35
+monitors that already exist in Datadog. Until `pulumi import` has told Pulumi
+they exist, a deploy would create 35 duplicates — duplicate pages to
+`#holy-shit-alarms` included.
+
+Adoption is a one-time local run. `pulumi import` reads Datadog and writes
+Pulumi state — it does not create, modify, or pause anything in Datadog. The
+Datadog keys are needed by `preview` as well: the provider validates every
+monitor against `/api/v1/monitor/validate` while diffing.
+
+```bash
+cd infra/stacks/monitoring
+export DD_API_KEY=... DD_APP_KEY=... DD_HOST=https://api.us5.datadoghq.com/
+pulumi stack select macro-inc/prod --create
+pulumi import --file import.json --out imported.generated.ts --protect --yes
+pulumi preview --diff
+```
+
+`--protect` matches the `adopted()` helper so state and code agree.
+`imported.generated.ts` is Pulumi's own rendering of the live monitors — the
+ground truth to reconcile against if the preview shows drift. Delete it after;
+it must not be committed (index.ts does not import it, and a second copy of
+every monitor would double-declare them).
+
+The gate is that preview:
+
+- **35 monitors + 1 synthetic test unchanged** — adoption is exact.
+- **2 monitors to create** (`ai-editing-*`) — expected, that's the new alerting.
+- **Any update, replace, or delete** — the hand-written code in `monitors/*.ts`
+ drifted from the live monitor. Reconcile against `imported.generated.ts`
+ before deploying. An `update` here silently changes who gets paged.
+
+Only then deploy: `pulumi up` locally, or **Actions → "deploy pulumi service"**
+with `pulumi-service-name: monitoring`, `environment: prod` (that action already
+exports the DD keys from repo secrets).
+
+## Why nothing here deploys by accident
+
+`.github/services-config.json` is the allowlist that drives `deploy_on_push.yml`
+and `pulumi_preview_pr.yml`. This stack is deliberately absent from it, so no
+push and no PR ever deploys it. It moves only by manual dispatch.
+
+Monitors are org-global rather than per-environment — a single monitor covers
+both the `-dev` and `-prod` services it queries. So there is one stack, `prod`,
+and no `dev` counterpart. (`deploy-pulumi-stack.yml` names the stack
+`macro-inc/<environment>`, hence `prod`.)
+
+## Credentials
+
+None to add. `deploy-cloud-storage-pulumi` already exports `DD_API_KEY`,
+`DD_APP_KEY`, and `DD_HOST=https://api.us5.datadoghq.com/`, which is all the
+Datadog provider reads. Do not set `datadog:validate: "false"` here — a missing
+key should fail the deploy rather than silently no-op.
+
+## Where transcription is most likely to be wrong
+
+The code was written from the Datadog API's own JSON, and `tsc` validates every
+field against the provider schema, so field *shapes* are right. Values that the
+provider models differently from the API are the risk, and these are the spots:
+
+- `monitors/logs.ts` `error-logs-overall` and `monitors/rum.ts`
+ `web-app-source-errors-prod` use `formula()` queries with `variables`. The API
+ returns one `compute` object per variable; the provider takes a `computes`
+ list. The API also carries `additionalProperties.storage: "hot"` on each
+ variable, which provider 4.68 cannot express at all.
+- `monitors/rds.ts` `rds-connection-anomaly` is an anomaly monitor, so its
+ `monitorThresholdWindows` are required rather than optional.
+- `synthetics.ts` reproduces Datadog's default notification template verbatim,
+ with backticks escaped for the template literal. Drift there changes alert
+ *text*, not alert behavior.
+- `onMissingData` values are the API's snake_case (`show_no_data`). Pulumi's
+ generated docs camelCase them; the provider forwards the string as-is.
+
+## Drift
+
+A code-managed monitor edited in the UI is reverted on the next deploy of this
+stack. Since that deploy is manual, drift shows up in `pulumi preview` before it
+is applied. Reach for `restrictedRoles` if convention stops being enough.
+
+## Adding a monitor
+
+Declare it in the matching `monitors/*.ts` with `new datadog.Monitor(...)`. Use
+the `adopted()` helper only for monitors that already exist in Datadog — it sets
+`protect`, which is there to stop a program that no longer declares a
+pre-existing monitor from deleting it.
+
+## AI-editing coverage
+
+The caller-side log alert matches `editing worker returned` for non-2xx HTTP
+responses and `editing worker request failed` for transport failures. The latter
+requires the updated `ReqwestEditingWorkerClient` to be deployed in the caller
+services; creating the monitor alone does not add that telemetry.
+
+Neither new alert detects successful responses that apply no edits, or failures
+during periods without editing requests. The end-to-end canary remains a
+follow-up.
diff --git a/infra/stacks/monitoring/adopted.ts b/infra/stacks/monitoring/adopted.ts
new file mode 100644
index 0000000000..5841e6a734
--- /dev/null
+++ b/infra/stacks/monitoring/adopted.ts
@@ -0,0 +1,17 @@
+import * as datadog from '@pulumi/datadog';
+
+/**
+ * A monitor that already exists in Datadog and was adopted into this stack by
+ * `pulumi import` (see README). `protect` is the point: these monitors predate
+ * the stack, several of them page on-call, and a program that no longer declares
+ * one would otherwise delete it on the next deploy. Protected resources fail the
+ * deploy instead.
+ *
+ * To retire a monitor, drop `protect` in its own commit, then delete it.
+ */
+export function adopted(
+ name: string,
+ args: datadog.MonitorArgs
+): datadog.Monitor {
+ return new datadog.Monitor(name, args, { protect: true });
+}
diff --git a/infra/stacks/monitoring/import.json b/infra/stacks/monitoring/import.json
new file mode 100644
index 0000000000..d2577b4933
--- /dev/null
+++ b/infra/stacks/monitoring/import.json
@@ -0,0 +1,40 @@
+{
+ "resources": [
+ { "type": "datadog:index/monitor:Monitor", "name": "macroai-error-logs", "id": "440085" },
+ { "type": "datadog:index/monitor:Monitor", "name": "macroapi-error-logs", "id": "440088" },
+ { "type": "datadog:index/monitor:Monitor", "name": "dss-errors-dev", "id": "1013923" },
+ { "type": "datadog:index/monitor:Monitor", "name": "dss-errors-prod", "id": "1062128" },
+ { "type": "datadog:index/monitor:Monitor", "name": "macro-cache-available-memory-low", "id": "1062487" },
+ { "type": "datadog:index/monitor:Monitor", "name": "alb-non-200-responses", "id": "1062605" },
+ { "type": "datadog:index/monitor:Monitor", "name": "rds-high-cpu", "id": "1062630" },
+ { "type": "datadog:index/monitor:Monitor", "name": "macrodb-low-burst-balance", "id": "1062636" },
+ { "type": "datadog:index/monitor:Monitor", "name": "cloud-storage-cache-high-cpu", "id": "1062685" },
+ { "type": "datadog:index/monitor:Monitor", "name": "cloud-storage-cache-low-memory", "id": "1062687" },
+ { "type": "datadog:index/monitor:Monitor", "name": "ecs-high-cpu", "id": "1062688" },
+ { "type": "datadog:index/monitor:Monitor", "name": "lambda-error-count", "id": "1065393" },
+ { "type": "datadog:index/monitor:Monitor", "name": "error-logs-overall", "id": "1067730" },
+ { "type": "datadog:index/monitor:Monitor", "name": "rds-proxy-connections", "id": "1076524" },
+ { "type": "datadog:index/monitor:Monitor", "name": "graphql-redis-connect", "id": "1082626" },
+ { "type": "datadog:index/monitor:Monitor", "name": "web-app-errors-staging", "id": "1177082" },
+ { "type": "datadog:index/monitor:Monitor", "name": "web-app-source-errors-prod", "id": "1189349" },
+ { "type": "datadog:index/monitor:Monitor", "name": "document-text-extractor-prod", "id": "1291765" },
+ { "type": "datadog:index/monitor:Monitor", "name": "document-text-extractor-dev", "id": "1291772" },
+ { "type": "datadog:index/monitor:Monitor", "name": "high-channel-invites", "id": "1333499" },
+ { "type": "datadog:index/monitor:Monitor", "name": "ses-bounce-rate", "id": "1333601" },
+ { "type": "datadog:index/monitor:Monitor", "name": "ses-complaint-rate", "id": "4178105" },
+ { "type": "datadog:index/monitor:Monitor", "name": "ses-high-email-sends", "id": "4757927" },
+ { "type": "datadog:index/monitor:Monitor", "name": "channel-invite-spam-dev", "id": "4876245" },
+ { "type": "datadog:index/monitor:Monitor", "name": "channel-invite-spam-prod", "id": "4876618" },
+ { "type": "datadog:index/monitor:Monitor", "name": "rds-storage-utilization", "id": "6120132" },
+ { "type": "datadog:index/monitor:Monitor", "name": "rds-connection-anomaly", "id": "6120237" },
+ { "type": "datadog:index/monitor:Monitor", "name": "alb-low-healthy-hosts", "id": "16891631" },
+ { "type": "datadog:index/monitor:Monitor", "name": "alb-high-response-time", "id": "16891718" },
+ { "type": "datadog:index/monitor:Monitor", "name": "dynamodb-write-capacity", "id": "16917541" },
+ { "type": "datadog:index/monitor:Monitor", "name": "dynamodb-read-capacity", "id": "16917574" },
+ { "type": "datadog:index/monitor:Monitor", "name": "rds-low-freeable-memory", "id": "16918778" },
+ { "type": "datadog:index/monitor:Monitor", "name": "rds-high-iops", "id": "16918920" },
+ { "type": "datadog:index/monitor:Monitor", "name": "apm-get-channel-messages-p95", "id": "19271629" },
+ { "type": "datadog:index/monitor:Monitor", "name": "opensearch-storage-low", "id": "21836655" },
+ { "type": "datadog:index/syntheticsTest:SyntheticsTest", "name": "app-welcome-health-check", "id": "3xv-xcg-7kp" }
+ ]
+}
diff --git a/infra/stacks/monitoring/index.ts b/infra/stacks/monitoring/index.ts
new file mode 100644
index 0000000000..4f5f25dc38
--- /dev/null
+++ b/infra/stacks/monitoring/index.ts
@@ -0,0 +1,21 @@
+// Datadog monitors and synthetic tests.
+//
+// Every monitor in the org is declared here. See README.md before deploying —
+// the 35 monitors under `monitors/` are adopted, not created, and a program that
+// stops declaring one deletes it.
+
+// Adopted from the Datadog UI.
+import './monitors/apm';
+import './monitors/cache';
+import './monitors/compute';
+import './monitors/dynamodb';
+import './monitors/load-balancer';
+import './monitors/logs';
+import './monitors/opensearch';
+import './monitors/rds';
+import './monitors/rum';
+import './monitors/ses';
+import './synthetics';
+
+// Declared here first.
+import './monitors/ai-editing';
diff --git a/infra/stacks/monitoring/monitors/ai-editing.ts b/infra/stacks/monitoring/monitors/ai-editing.ts
new file mode 100644
index 0000000000..c32f166e5e
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/ai-editing.ts
@@ -0,0 +1,86 @@
+// AI document editing.
+//
+// New monitors, not adopted from the UI, so they are not `protect`ed — they can
+// be tuned or removed from code freely.
+//
+// Two independent angles, because each misses what the other catches. The APM
+// monitor sees the worker's own view of its requests; the log monitor sees
+// the callers' view, so it still fires when the worker is unreachable and
+// reporting no telemetry at all.
+//
+// Neither covers "the worker answers 200 but silently produces no edits", and
+// both are blind while nobody is editing. That needs the hourly canary against
+// a fixed document, which is a follow-up: POST /edit requires a short-lived
+// `documentToken` (macro_sync_service_jwt), so a probe needs somewhere to mint
+// one.
+import * as datadog from '@pulumi/datadog';
+
+// The worker's entry span is `http.server.request` with resource_name `POST`.
+// Custom spans like `edit.session` do NOT produce trace metrics — Datadog only
+// generates them for entry spans, so a monitor on `trace.Internal{...}` here
+// would never fire.
+export const aiEditingWorkerErrors = new datadog.Monitor(
+ 'ai-editing-worker-errors',
+ {
+ name: '[PROD] AI editing worker returning errors',
+ type: 'query alert',
+ query:
+ 'sum(last_1h):(sum:trace.http.server.request.errors{service:ai-editing-worker,env:production}.as_count() / sum:trace.http.server.request.hits{service:ai-editing-worker,env:production}.as_count()) * 100 > 50',
+ message: `More than {{threshold}}% of AI editing requests are failing.
+
+The worker answers 502 when an edit session throws and 499 when the client
+cancels; only the former counts as an error here.
+
+- [Worker traces](https://us5.datadoghq.com/apm/traces?query=service%3Aai-editing-worker%20env%3Aproduction%20status%3Aerror)
+- Check provider outages first: the supervisor chain falls back Anthropic ->
+ OpenAI, so a single provider failing shows up as latency, not errors.
+
+@slack-monitoring`,
+ tags: ['env:prod', 'service:ai-editing-worker', 'product:apm'],
+ monitorThresholds: { critical: '50', warning: '20' },
+ // Ratio of two count metrics: no traffic means no data, which must not
+ // alert. Quiet-hours coverage is the canary's job, not this monitor's.
+ onMissingData: 'default',
+ evaluationDelay: 60,
+ includeTags: true,
+ notifyAudit: false,
+ }
+);
+
+// ReqwestEditingWorkerClient::edit reports "editing worker returned" for
+// non-2xx responses and "editing worker request failed" for transport failures
+// (crates/documents/src/outbound/editing_worker_client.rs).
+//
+// The callers are document-cognition-service (the EditDocument tool, 174 of
+// these in the 30 days to 2026-08-31) and cloud-storage-service (7). Note the
+// service tags carry no `-prod` suffix and the status is `warn`, unlike the
+// older log monitors in logs.ts — matching on `service:*-prod` here finds
+// nothing. agent-schedule-service is excluded: its hits are agent transcripts
+// that quote the error text rather than reports of it.
+//
+// Thresholds come from that 30-day window: the worst normal hour was 12, so
+// 25/hour is roughly double the known-bad baseline rather than a guess.
+export const aiEditingRequestsFailing = new datadog.Monitor(
+ 'ai-editing-requests-failing',
+ {
+ name: '[PROD] AI editing requests failing for callers',
+ type: 'log alert',
+ query:
+ 'logs("env:prod service:(document-cognition-service OR cloud-storage-service) (\\"editing worker returned\\" OR \\"editing worker request failed\\")").index("*").rollup("count").last("1h") > 25',
+ message: `Callers are getting errors back from the AI editing worker.
+
+Unlike the APM monitor this fires even when the worker is unreachable, since it
+reads the callers' logs rather than the worker's telemetry. The common cause
+historically is \`initial sync failed: timeout\` — the worker could not open the
+document over the sync service.
+
+@slack-monitoring`,
+ tags: ['env:prod', 'service:ai-editing-worker'],
+ monitorThresholds: { critical: '25', warning: '12' },
+ onMissingData: 'default',
+ includeTags: false,
+ notifyAudit: false,
+ enableLogsSample: true,
+ groupbySimpleMonitor: false,
+ }
+);
diff --git a/infra/stacks/monitoring/monitors/apm.ts b/infra/stacks/monitoring/monitors/apm.ts
new file mode 100644
index 0000000000..78577ffcb6
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/apm.ts
@@ -0,0 +1,49 @@
+// APM latency.
+//
+// Adopted from monitors created by hand in the Datadog UI. Do not edit a
+// monitor here and in the UI at the same time — this file wins on deploy.
+import { adopted } from '../adopted';
+
+// https://us5.datadoghq.com/monitors/19271629
+adopted('apm-get-channel-messages-p95', {
+ name: 'High p95 Latency – get_channel_messages (cloud-storage-service prod)',
+ type: 'query alert',
+ query:
+ 'percentile(last_15m):p95:trace.Internal{service:cloud-storage-service,resource_name:get_channel_messages,env:prod} > 0.1',
+ message: `## ⚠️ High p95 Latency on get_channel_messages
+
+**Service:** cloud-storage-service
+**Resource:** get_channel_messages
+**Environment:** prod
+
+### What's happening?
+p95 response time has exceeded **100ms** (threshold). During the Apr 17 incident this reached 322ms.
+
+### What to check?
+1. Check for recent ECS deployments on \`cloud-storage-service\` — the Apr 17 spike was caused by a bad task version
+2. Look for \`idle_ns >> busy_ns\` in slow spans — indicates DB wait / connection pool exhaustion
+3. Check channels \`0195ceb6-ec2e-7023-80e4-6e084fa6cccd\` and \`0195cea4-5491-77f9-9224-1291fdb8f150\` (repeatedly affected in the last incident)
+4. Review the \`Sort(CreatedAt)\` query path on backward pagination (\`direction: Older\`)
+
+### Useful Links
+- [Slow spans explorer](https://us5.datadoghq.com/apm/traces?query=service%3Acloud-storage-service+resource_name%3Aget_channel_messages+env%3Aprod+%40duration%3A%3E100000000)
+- [Investigation notebook](https://us5.datadoghq.com/notebook/277323)`,
+ priority: '5',
+ tags: [
+ 'service:cloud-storage-service',
+ 'env:prod',
+ 'resource:get_channel_messages',
+ 'product:apm',
+ 'team:hutch',
+ ],
+ monitorThresholds: {
+ critical: '0.1',
+ warning: '0.05',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: true,
+ newHostDelay: 300,
+ evaluationDelay: 60,
+ requireFullWindow: false,
+});
diff --git a/infra/stacks/monitoring/monitors/cache.ts b/infra/stacks/monitoring/monitors/cache.ts
new file mode 100644
index 0000000000..c40df50b75
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/cache.ts
@@ -0,0 +1,73 @@
+// ElastiCache and MemoryDB.
+//
+// Adopted from monitors created by hand in the Datadog UI. Do not edit a
+// monitor here and in the UI at the same time — this file wins on deploy.
+import { adopted } from '../adopted';
+
+// https://us5.datadoghq.com/monitors/1062487
+adopted('macro-cache-available-memory-low', {
+ name: '[PROD] Macro Cache Available Memory Low',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.elasticache.database_memory_usage_percentage{name:macro-cache-prod*} > 90',
+ message: `{{#is_alert}}
+@slack-holy-shit-alarms
+{{/is_alert}}
+{{#is_alert_recovery}}
+@slack-holy-shit-alarms
+{{/is_alert_recovery}}
+
+{{#is_warning_recovery}}
+@slack-monitoring
+{{/is_warning_recovery}}
+{{#is_warning}}
+@slack-monitoring
+{{/is_warning}}`,
+ monitorThresholds: {
+ critical: '90',
+ warning: '80',
+ },
+ notifyNoData: false,
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ evaluationDelay: 900,
+});
+
+// https://us5.datadoghq.com/monitors/1062685
+adopted('cloud-storage-cache-high-cpu', {
+ name: '[PROD] Cloud Storage Cache High CPU Utilization',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.memorydb.cpuutilization{clustername:cloud-storage-cache-memorydb-prod} > 90',
+ message:
+ '{{#is_alert}} @slack-holy-shit-alarms {{/is_alert}} {{#is_alert_recovery}} @slack-holy-shit-alarms {{/is_alert_recovery}}',
+ monitorThresholds: {
+ critical: '90',
+ warning: '80',
+ },
+ notifyNoData: false,
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ evaluationDelay: 900,
+});
+
+// https://us5.datadoghq.com/monitors/1062687
+adopted('cloud-storage-cache-low-memory', {
+ name: '[PROD] Cloud Storage Cache Low Available Memory',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.memorydb.database_memory_usage_percentage{*} > 90',
+ message:
+ '{{#is_alert}} @slack-holy-shit-alarms {{/is_alert}} {{#is_alert_recovery}} @slack-holy-shit-alarms {{/is_alert_recovery}}',
+ monitorThresholds: {
+ critical: '90',
+ warning: '60',
+ },
+ notifyNoData: false,
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ evaluationDelay: 900,
+});
diff --git a/infra/stacks/monitoring/monitors/compute.ts b/infra/stacks/monitoring/monitors/compute.ts
new file mode 100644
index 0000000000..350e6e6c9b
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/compute.ts
@@ -0,0 +1,87 @@
+// ECS and Lambda.
+//
+// Adopted from monitors created by hand in the Datadog UI. Do not edit a
+// monitor here and in the UI at the same time — this file wins on deploy.
+import { adopted } from '../adopted';
+
+// https://us5.datadoghq.com/monitors/1062688
+adopted('ecs-high-cpu', {
+ name: '[PROD] High cpu usage across ECS Clusters',
+ type: 'query alert',
+ query:
+ 'avg(last_5m):sum:ecs.fargate.cpu.percent{(ecs_cluster:macro-cluster-prod OR ecs_cluster:cloud-storage-cluster-prod OR ecs_cluster:macroai-cluster-prod-4cbe721)} by {task_name} > 500',
+ message: `{{task_name.name}} is experiencing high cpu usage in ECS
+
+@slack-monitoring`,
+ monitorThresholds: {
+ critical: '500',
+ warning: '400',
+ },
+ onMissingData: 'show_no_data',
+ notifyAudit: false,
+ includeTags: false,
+ newGroupDelay: 0,
+});
+
+// https://us5.datadoghq.com/monitors/1065393
+adopted('lambda-error-count', {
+ name: '[PROD] Lambda Increased Error Count',
+ type: 'query alert',
+ query:
+ 'sum(last_1h):sum:aws.lambda.errors{(level:prod OR environment:prod ) AND NOT functionname:ws-api-gateway-authorizer-lambda-prod} by {functionname}.as_rate() > 500',
+ message: `{{#is_alert}}@slack-holy-shit-alarms {{/is_alert}}
+{{#is_alert_recovery}}@slack-holy-shit-alarms{{/is_alert_recovery}}
+
+@slack-monitoring`,
+ monitorThresholds: {
+ critical: '500',
+ warning: '100',
+ },
+ onMissingData: 'show_no_data',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 60,
+ evaluationDelay: 900,
+});
+
+// https://us5.datadoghq.com/monitors/1291765
+adopted('document-text-extractor-prod', {
+ name: '[PROD] Document Text Extractor Failing',
+ type: 'query alert',
+ query:
+ 'sum(last_4h):(sum:aws.lambda.errors{resource:document-text-extractor-prod}.as_count() / sum:aws.lambda.invocations{resource:document-text-extractor-prod}.as_count()) * 100 > 80',
+ message: `@[email protected]
+@slack-monitoring
+Document text extractor failing`,
+ monitorThresholds: {
+ critical: '80',
+ warning: '50',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ evaluationDelay: 900,
+ renotifyInterval: 0,
+});
+
+// https://us5.datadoghq.com/monitors/1291772
+adopted('document-text-extractor-dev', {
+ name: '[DEV] Document Text Extractor Failing',
+ type: 'query alert',
+ query:
+ 'sum(last_4h):(sum:aws.lambda.errors{resource:document-text-extractor-dev}.as_count() / sum:aws.lambda.invocations{resource:document-text-extractor-dev}.as_count()) * 100 > 80',
+ message: `@[email protected]
+@slack-monitoring
+Document text extractor failing`,
+ monitorThresholds: {
+ critical: '80',
+ warning: '50',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ evaluationDelay: 900,
+ renotifyInterval: 0,
+});
diff --git a/infra/stacks/monitoring/monitors/dynamodb.ts b/infra/stacks/monitoring/monitors/dynamodb.ts
new file mode 100644
index 0000000000..067f457696
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/dynamodb.ts
@@ -0,0 +1,43 @@
+// DynamoDB consumed capacity.
+//
+// Adopted from monitors created by hand in the Datadog UI. Do not edit a
+// monitor here and in the UI at the same time — this file wins on deploy.
+import { adopted } from '../adopted';
+
+// https://us5.datadoghq.com/monitors/16917541
+adopted('dynamodb-write-capacity', {
+ name: 'Dynamodb ConsumedWriteCapacityUnits High for {{tablename.name}}',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.dynamodb.consumed_write_capacity_units{*} by {tablename} > 500',
+ message: `@[email protected]
+ monitorThresholds: {
+ critical: '500',
+ warning: '300',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 60,
+ evaluationDelay: 900,
+});
+
+// https://us5.datadoghq.com/monitors/16917574
+adopted('dynamodb-read-capacity', {
+ name: 'Dynamodb ConsumedReadCapacityUnits High for {{tablename.name}}',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.dynamodb.consumed_read_capacity_units{*} by {tablename} > 20',
+ message: `@[email protected]
+ monitorThresholds: {
+ critical: '20',
+ warning: '10',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 60,
+ evaluationDelay: 900,
+});
diff --git a/infra/stacks/monitoring/monitors/load-balancer.ts b/infra/stacks/monitoring/monitors/load-balancer.ts
new file mode 100644
index 0000000000..dddff183ed
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/load-balancer.ts
@@ -0,0 +1,80 @@
+// Application load balancers.
+//
+// Adopted from monitors created by hand in the Datadog UI. Do not edit a
+// monitor here and in the UI at the same time — this file wins on deploy.
+import { adopted } from '../adopted';
+
+// https://us5.datadoghq.com/monitors/1062605
+adopted('alb-non-200-responses', {
+ name: '[PROD] Load Balancer Non-200 Responses',
+ type: 'query alert',
+ query:
+ 'sum(last_1h):(sum:aws.applicationelb.httpcode_elb_5xx{level:prod OR environment:prod*} by {loadbalancer}.as_count() / sum:aws.applicationelb.request_count{level:prod OR environment:prod*} by {loadbalancer}.as_count()) * 100 > 90',
+ message: `Detected a high number of non-200 responses for ALB {{loadbalancer.name}}
+
+
+{{#is_alert}}
+@slack-holy-shit-alarms
+{{/is_alert}}
+{{#is_alert_recovery}}
+@slack-holy-shit-alarms
+{{/is_alert_recovery}}
+
+@slack-monitoring
+
+
+@slack-monitoring`,
+ monitorThresholds: {
+ critical: '90',
+ warning: '60',
+ },
+ onMissingData: 'show_no_data',
+ notifyAudit: false,
+ includeTags: false,
+ newGroupDelay: 0,
+ evaluationDelay: 900,
+ renotifyInterval: 0,
+ requireFullWindow: false,
+});
+
+// https://us5.datadoghq.com/monitors/16891631
+adopted('alb-low-healthy-hosts', {
+ name: 'Application load balancer low healthy host count',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):abs(avg:aws.applicationelb.healthy_host_count{*} by {loadbalancer}) <= 0.25',
+ message: `The application load balancer for {{loadbalancer.name}} has low healthy host count.
+
+
+ monitorThresholds: {
+ critical: '0.25',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 60,
+ evaluationDelay: 900,
+});
+
+// https://us5.datadoghq.com/monitors/16891718
+adopted('alb-high-response-time', {
+ name: 'High average response time for alb {{loadbalancer.name}}',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.applicationelb.target_response_time.average{! name:proxy-service-alb-prod-153de67 , ! name:store-service-alb-prod-4db6301} by {loadbalancer} > 3',
+ message: `Loadbalancer {{loadbalancer.name}} has a high average response time
+
+ monitorThresholds: {
+ critical: '3',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 60,
+ evaluationDelay: 900,
+});
diff --git a/infra/stacks/monitoring/monitors/logs.ts b/infra/stacks/monitoring/monitors/logs.ts
new file mode 100644
index 0000000000..dac8a31710
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/logs.ts
@@ -0,0 +1,235 @@
+// Log-based error-rate monitors.
+//
+// Adopted from monitors created by hand in the Datadog UI. Do not edit a
+// monitor here and in the UI at the same time — this file wins on deploy.
+import { adopted } from '../adopted';
+
+// https://us5.datadoghq.com/monitors/440085
+adopted('macroai-error-logs', {
+ name: '[MACROAI] Increase in Error logs',
+ type: 'log alert',
+ query:
+ 'logs("service:(macroai-query-service-prod OR macroai-store-service-prod OR macroai-proxy-service-prod) status:error").index("*").rollup("count").last("5m") > 2500',
+ message: `@[email protected]
+@slack-monitoring`,
+ tags: ['macro-ai'],
+ monitorThresholds: {
+ critical: '2500',
+ warning: '1000',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ renotifyInterval: 0,
+ enableLogsSample: true,
+ groupbySimpleMonitor: false,
+});
+
+// https://us5.datadoghq.com/monitors/440088
+adopted('macroapi-error-logs', {
+ name: '[MACROAPI] Increase in Error Logs',
+ type: 'log alert',
+ query:
+ 'logs("service:macro-gql-service-prod status:error").index("*").rollup("count").last("5m") > 50',
+ message: '@slack-monitoring',
+ monitorThresholds: {
+ critical: '50',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ enableLogsSample: false,
+ groupbySimpleMonitor: false,
+});
+
+// https://us5.datadoghq.com/monitors/1013923
+adopted('dss-errors-dev', {
+ name: '[DEV] Increased number of errors for DSS',
+ type: 'log alert',
+ query:
+ 'logs("service:cloud-storage-service-dev ERROR*").index("*").rollup("count").last("5m") > 200',
+ message: '@slack-monitoring',
+ monitorThresholds: {
+ critical: '200',
+ warning: '100',
+ },
+ onMissingData: 'default',
+ notifyAudit: true,
+ includeTags: false,
+ newHostDelay: 300,
+ enableLogsSample: true,
+ groupbySimpleMonitor: false,
+});
+
+// https://us5.datadoghq.com/monitors/1062128
+adopted('dss-errors-prod', {
+ name: '[PROD] Increased number of errors for DSS',
+ type: 'log alert',
+ query:
+ 'logs("service:cloud-storage-service-prod ERROR*").index("*").rollup("count").last("5m") > 150',
+ message: '@slack-monitoring',
+ monitorThresholds: {
+ critical: '150',
+ warning: '50',
+ },
+ onMissingData: 'default',
+ notifyAudit: true,
+ includeTags: false,
+ newHostDelay: 300,
+ enableLogsSample: true,
+ groupbySimpleMonitor: false,
+});
+
+// https://us5.datadoghq.com/monitors/1067730
+adopted('error-logs-overall', {
+ name: '[PROD] Increase in Error Logs',
+ type: 'log alert',
+ query: 'formula("query - query1").last("5m") > 25000',
+ message: `Increase in overall count of error logs across production services.
+Please view datadog to checkout what specific items may be causing the issues.
+
+{{#is_alert}}
+@slack-holy-shit-alarms
+{{/is_alert}}
+{{#is_alert_recovery}}
+@slack-holy-shit-alarms
+{{/is_alert_recovery}}
+
+@slack-monitoring`,
+ priority: '1',
+ tags: ['prod'],
+ monitorThresholds: {
+ critical: '25000',
+ warning: '10000',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ enableLogsSample: true,
+ groupbySimpleMonitor: false,
+ variables: {
+ eventQueries: [
+ {
+ name: 'query',
+ dataSource: 'logs',
+ computes: [
+ {
+ aggregation: 'count',
+ metric: 'count',
+ },
+ ],
+ search: {
+ query:
+ '(service:*-prod OR (service:document-processor* @env:prod)) error',
+ },
+ },
+ {
+ name: 'query1',
+ dataSource: 'logs',
+ computes: [
+ {
+ aggregation: 'count',
+ },
+ ],
+ indexes: ['*'],
+ search: {
+ query: 'service:macroai-proxy-service-prod error',
+ },
+ },
+ ],
+ },
+});
+
+// https://us5.datadoghq.com/monitors/1082626
+adopted('graphql-redis-connect', {
+ name: '[MACROAPI] GraphQL could not connect to Redis',
+ type: 'log alert',
+ query:
+ 'logs("service:macro-gql-service-prod status:error \\"Failed to reconnect to Redis cache at URL*\\"").index("*").rollup("count").last("5m") >= 1',
+ message: '@slack-monitoring',
+ priority: '3',
+ tags: ['prod'],
+ monitorThresholds: {
+ critical: '1',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ enableLogsSample: true,
+ groupbySimpleMonitor: false,
+});
+
+// https://us5.datadoghq.com/monitors/1333499
+adopted('high-channel-invites', {
+ name: 'High Channel Invites',
+ type: 'log alert',
+ query:
+ 'logs("service:notification-service-prod @notification_event_type:channel_invite").index("*").rollup("count", "@sender_id").last("1h") > 100',
+ message: `@slack-monitoring
+High number of channel invites
+
+Investigate whether 1 user is causing this with this log query
+
+\`service:notification-service-prod @notification_event_type:channel_invite\``,
+ monitorThresholds: {
+ critical: '100',
+ warning: '50',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ enableLogsSample: false,
+ groupbySimpleMonitor: false,
+});
+
+// https://us5.datadoghq.com/monitors/4876245
+adopted('channel-invite-spam-dev', {
+ name: '[Channel Invites] [DEV] Potential Email Spam on Channel Invites',
+ type: 'log alert',
+ query:
+ 'logs("service:notification-service-dev message:\\"processing message\\" @notification_event_type:channel_invite").index("*").rollup("count").by("@sender_id").last("1h") > 10',
+ message: `A user is generating a high number of channel invites. This could mean they are spamming users with invite emails.
+
+@slack-monitoring
+
+ monitorThresholds: {
+ critical: '10',
+ warning: '5',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newGroupDelay: 0,
+ enableLogsSample: false,
+ groupbySimpleMonitor: false,
+});
+
+// https://us5.datadoghq.com/monitors/4876618
+adopted('channel-invite-spam-prod', {
+ name: '[Channel Invites] [PROD] Potential Email Spam on Channel Invites',
+ type: 'log alert',
+ query:
+ 'logs("service:notification-service-prod message:\\"processing message\\" @notification_event_type:channel_invite").index("*").rollup("count").by("@sender_id").last("1h") > 10',
+ message: `A user is generating a high number of channel invites. This could mean they are spamming users with invite emails.
+
+@slack-monitoring
+
+ monitorThresholds: {
+ critical: '10',
+ warning: '5',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newGroupDelay: 0,
+ enableLogsSample: false,
+ groupbySimpleMonitor: false,
+ groupRetentionDuration: '1h',
+});
diff --git a/infra/stacks/monitoring/monitors/opensearch.ts b/infra/stacks/monitoring/monitors/opensearch.ts
new file mode 100644
index 0000000000..022cf98aad
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/opensearch.ts
@@ -0,0 +1,58 @@
+// OpenSearch cluster.
+//
+// Adopted from monitors created by hand in the Datadog UI. Do not edit a
+// monitor here and in the UI at the same time — this file wins on deploy.
+import { adopted } from '../adopted';
+
+// https://us5.datadoghq.com/monitors/21836655
+adopted('opensearch-storage-low', {
+ name: '[AWS] Opensearch Storage Decreasing',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.es.free_storage_space.minimum{region:us-east-1} by {environment} < 10000',
+ message: `{{#is_alert}}
+🚨 CRITICAL — Low Free Opensearch Storage
+
+Environment: {{environment}}
+Storage utilization: {{value}}%
+Critical threshold: {{threshold}}%
+
+Opensearch storage utilization has reached a critical level. Investigate storage growth or increase the allocated storage.
+
+@webhook-macro-alert-hook
+{{/is_alert}}
+
+{{#is_warning}}
+⚠️ WARNING — Low Free Opensearch Storage
+
+Environment: {{environment}}
+Storage utilization: {{value}}%
+Warning threshold: {{warn_threshold}}%
+
+Opensearch storage utilization is approaching the critical threshold.
+
+@webhook-macro-alert-hook
+{{/is_warning}}
+
+{{#is_recovery}}
+✅ RECOVERED — Opensearch Storage Normal
+
+Environment: {{environment}}
+Storage utilization: {{value}}%
+
+Opensearch storage utilization has returned to an acceptable level.
+
+@webhook-macro-alert-hook
+{{/is_recovery}}
+
+ monitorThresholds: {
+ critical: '10000',
+ warning: '20000',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 60,
+ evaluationDelay: 900,
+});
diff --git a/infra/stacks/monitoring/monitors/rds.ts b/infra/stacks/monitoring/monitors/rds.ts
new file mode 100644
index 0000000000..6e81abc719
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/rds.ts
@@ -0,0 +1,192 @@
+// MacroDB / RDS health.
+//
+// Adopted from monitors created by hand in the Datadog UI. Do not edit a
+// monitor here and in the UI at the same time — this file wins on deploy.
+import { adopted } from '../adopted';
+
+// https://us5.datadoghq.com/monitors/1062630
+adopted('rds-high-cpu', {
+ name: '[PROD] RDS High CPU Utilization',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):sum:aws.rds.cpuutilization{name:*-prod} by {name}.weighted() >= 95',
+ message: `@[email protected]
+@webhook-macro-alert-hook
+
+RDS DB {{name.name}} is experiencing high CPU utilization.`,
+ monitorThresholds: {
+ critical: '95',
+ warning: '80',
+ },
+ onMissingData: 'show_no_data',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 0,
+ evaluationDelay: 900,
+ renotifyInterval: 0,
+ requireFullWindow: false,
+});
+
+// https://us5.datadoghq.com/monitors/1062636
+adopted('macrodb-low-burst-balance', {
+ name: '[PROD] MacroDB Low Burst Balance',
+ type: 'query alert',
+ query: 'avg(last_1h):avg:aws.rds.burst_balance{name:macro-db-prod} <= 10',
+ message: '@slack-monitoring',
+ monitorThresholds: {
+ critical: '10',
+ warning: '25',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ evaluationDelay: 900,
+});
+
+// https://us5.datadoghq.com/monitors/1076524
+adopted('rds-proxy-connections', {
+ name: '[PROD] Excessive concurrent connections in rds proxy',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.rds.proxy.database_connections{proxyname:macrodb-proxy-prod} > 500',
+ message: `We have excessive concurrent connections in the rds proxy. This can lead to slow database calls and potential outage if other services can not secure a connection
+
+@slack-holy-shit-alarms`,
+ monitorThresholds: {
+ critical: '500',
+ },
+ notifyNoData: false,
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ evaluationDelay: 900,
+});
+
+// https://us5.datadoghq.com/monitors/6120132
+adopted('rds-storage-utilization', {
+ name: '[AWS] RDS Storage utilization is high',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):100 - ((avg:aws.rds.free_storage_space{*} by {dbinstanceidentifier,engine} / avg:aws.rds.total_storage_space{*} by {dbinstanceidentifier,engine}) * 100) > 90',
+ message: `{{#is_alert}}
+🚨 CRITICAL — RDS Storage High
+
+Instance: {{dbinstanceidentifier.name}}
+Engine: {{engine.name}}
+Storage utilization: {{value}}%
+Critical threshold: {{threshold}}%
+
+RDS storage utilization has reached a critical level. Investigate storage growth or increase the allocated storage.
+
+@webhook-macro-alert-hook
+{{/is_alert}}
+
+{{#is_warning}}
+⚠️ WARNING — RDS Storage High
+
+Instance: {{dbinstanceidentifier.name}}
+Engine: {{engine.name}}
+Storage utilization: {{value}}%
+Warning threshold: {{warn_threshold}}%
+
+RDS storage utilization is approaching the critical threshold.
+
+@webhook-macro-alert-hook
+{{/is_warning}}
+
+{{#is_recovery}}
+✅ RECOVERED — RDS Storage Normal
+
+Instance: {{dbinstanceidentifier.name}}
+Engine: {{engine.name}}
+Storage utilization: {{value}}%
+
+RDS storage utilization has returned to an acceptable level.
+
+@webhook-macro-alert-hook
+{{/is_recovery}}
+
+ tags: ['integration:amazon_rds'],
+ monitorThresholds: {
+ critical: '90',
+ warning: '85',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 60,
+ evaluationDelay: 900,
+ renotifyInterval: 0,
+ requireFullWindow: false,
+});
+
+// https://us5.datadoghq.com/monitors/6120237
+adopted('rds-connection-anomaly', {
+ name: '[AWS] RDS Anomaly in database connections',
+ type: 'query alert',
+ query:
+ "avg(last_1d):anomalies(avg:aws.rds.database_connections{name:*-prod} by {dbinstanceidentifier}, 'basic', 2, direction='above', interval=300, alert_window='last_1h', count_default_zero='true') >= 0.99",
+ message: `The number of connections for RDS database {{dbinstanceidentifier.name}} is outside of the typical range.
+
+@slack-monitoring`,
+ tags: ['integration:amazon_rds'],
+ monitorThresholds: {
+ critical: '0.99',
+ criticalRecovery: '0',
+ },
+ monitorThresholdWindows: {
+ recoveryWindow: 'last_15m',
+ triggerWindow: 'last_1h',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 60,
+ evaluationDelay: 900,
+ renotifyInterval: 0,
+ requireFullWindow: false,
+});
+
+// https://us5.datadoghq.com/monitors/16918778
+adopted('rds-low-freeable-memory', {
+ name: 'Low freeable memory for {{database_instance.name}}',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.rds.freeable_memory{environment:prod OR database_instance:macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com} by {database_instance} < 2000000000',
+ message: `@[email protected]
+ monitorThresholds: {
+ critical: '2000000000',
+ warning: '3000000000',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 60,
+ evaluationDelay: 900,
+});
+
+// https://us5.datadoghq.com/monitors/16918920
+adopted('rds-high-iops', {
+ name: 'Database high iops for {{database_instance.name}}',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.rds.write_iops{environment:prod OR database_instance:macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com} by {database_instance}.as_rate() + avg:aws.rds.read_iops{environment:prod OR database_instance:macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com} by {database_instance}.as_rate() > 10000',
+ message: `@[email protected]
+
+@webhook-alarm_webhook_test`,
+ monitorThresholds: {
+ critical: '10000',
+ warning: '9000',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 60,
+ evaluationDelay: 900,
+});
diff --git a/infra/stacks/monitoring/monitors/rum.ts b/infra/stacks/monitoring/monitors/rum.ts
new file mode 100644
index 0000000000..8c8caae944
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/rum.ts
@@ -0,0 +1,84 @@
+// Real User Monitoring alerts on the web app.
+//
+// Adopted from monitors created by hand in the Datadog UI. Do not edit a
+// monitor here and in the UI at the same time — this file wins on deploy.
+import { adopted } from '../adopted';
+
+// https://us5.datadoghq.com/monitors/1177082
+adopted('web-app-errors-staging', {
+ name: '[STAGING] Web-App Increased Error Count',
+ type: 'rum alert',
+ query:
+ 'rum("@type:error @application.name:\\"Web App\\" env:production @view.url_host:staging.macro.com").rollup("count").last("5m") > 90',
+ message: '@slack-monitoring',
+ monitorThresholds: {
+ critical: '90',
+ warning: '50',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ enableLogsSample: false,
+ groupbySimpleMonitor: false,
+});
+
+// https://us5.datadoghq.com/monitors/1189349
+adopted('web-app-source-errors-prod', {
+ name: '[PROD] Web-App Source Error',
+ type: 'rum alert',
+ query: 'formula("(1 - (query - query1) / query) * 100").last("1h") > 90',
+ message: `{{value}} % of all views in Web App are throwing errors from source.
+
+@slack-monitoring
+{{#is_alert}}@slack-holy-shit-alarms {{/is_alert}}`,
+ priority: '2',
+ tags: ['prod'],
+ monitorThresholds: {
+ critical: '90',
+ warning: '50',
+ },
+ onMissingData: 'show_no_data',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ renotifyInterval: 60,
+ renotifyStatuses: ['alert'],
+ notificationPresetName: 'hide_all',
+ enableLogsSample: false,
+ groupbySimpleMonitor: false,
+ variables: {
+ eventQueries: [
+ {
+ name: 'query',
+ dataSource: 'rum',
+ computes: [
+ {
+ aggregation: 'cardinality',
+ metric: '@view.id',
+ },
+ ],
+ indexes: ['*'],
+ search: {
+ query:
+ '@type:view (@application.name:"Web App" OR service:web-app) env:production @view.url_host:(macro.com OR www.macro.com)',
+ },
+ },
+ {
+ name: 'query1',
+ dataSource: 'rum',
+ computes: [
+ {
+ aggregation: 'cardinality',
+ metric: '@view.id',
+ },
+ ],
+ indexes: ['*'],
+ search: {
+ query:
+ '@type:error (@application.name:"Web App" OR service:web-app) env:production @view.url_host:(macro.com OR www.macro.com) @error.source:source',
+ },
+ },
+ ],
+ },
+});
diff --git a/infra/stacks/monitoring/monitors/ses.ts b/infra/stacks/monitoring/monitors/ses.ts
new file mode 100644
index 0000000000..ce427ff29b
--- /dev/null
+++ b/infra/stacks/monitoring/monitors/ses.ts
@@ -0,0 +1,78 @@
+// SES sending reputation.
+//
+// Adopted from monitors created by hand in the Datadog UI. Do not edit a
+// monitor here and in the UI at the same time — this file wins on deploy.
+import { adopted } from '../adopted';
+
+// https://us5.datadoghq.com/monitors/1333601
+adopted('ses-bounce-rate', {
+ name: '[SES] Bounce Rate Increasing',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.ses.reputation_bounce_rate{aws_account:569036502058} > 0.05',
+ message: `SES Bounce Rate Increasing.
+
+{{#is_alert}} @slack-holy-shit-alarms {{/is_alert}} {{#is_alert_recovery}} @slack-holy-shit-alarms {{/is_alert_recovery}}
+
+@slack-monitoring`,
+ monitorThresholds: {
+ critical: '0.05',
+ warning: '0.03',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ evaluationDelay: 900,
+});
+
+// https://us5.datadoghq.com/monitors/4178105
+adopted('ses-complaint-rate', {
+ name: '[SES] Complaint Rate Increasing',
+ type: 'query alert',
+ query:
+ 'avg(last_1h):avg:aws.ses.reputation_complaint_rate{aws_account:569036502058} > 0.005',
+ message: `SES Bounce Rate Increasing. Action needs to be taken to prevent account suspension.
+
+{{#is_warning}}@slack-monitoring{{/is_warning}}
+{{#is_warning_recovery}}@slack-monitoring {{/is_warning_recovery}}
+
+{{#is_alert}}@slack-holy-shit-alarms {{/is_alert}}
+{{#is_alert_recovery}}@slack-holy-shit-alarms @slack-monitoring {{/is_alert_recovery}}
+
+ monitorThresholds: {
+ critical: '0.005',
+ warning: '0.001',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: false,
+ newHostDelay: 300,
+ evaluationDelay: 900,
+});
+
+// https://us5.datadoghq.com/monitors/4757927
+adopted('ses-high-email-sends', {
+ name: '[SES] HIGH EMAIL SENDS',
+ type: 'query alert',
+ query: 'sum(last_1h):avg:aws.ses.send.sum{*} by {aws_account} > 250',
+ message: `SES is experiencing high email sends. This could be related to a potential attack and may require users to be blocked.
+
+{{#is_warning}}@slack-monitoring{{/is_warning}}
+{{#is_warning_recovery}}@slack-monitoring {{/is_warning_recovery}}
+
+{{#is_alert}}@slack-holy-shit-alarms {{/is_alert}}
+{{#is_alert_recovery}}@slack-holy-shit-alarms @slack-monitoring {{/is_alert_recovery}}
+
+ monitorThresholds: {
+ critical: '250',
+ warning: '200',
+ },
+ onMissingData: 'default',
+ notifyAudit: false,
+ includeTags: true,
+ newGroupDelay: 60,
+ evaluationDelay: 900,
+});
diff --git a/infra/stacks/monitoring/package.json b/infra/stacks/monitoring/package.json
new file mode 100644
index 0000000000..43e0c93469
--- /dev/null
+++ b/infra/stacks/monitoring/package.json
@@ -0,0 +1,14 @@
+{
+ "name": "monitoring-stack",
+ "version": "0.0.0",
+ "private": true,
+ "license": "MIT",
+ "main": "index.ts",
+ "scripts": {
+ "preview:prod": "pulumi stack select macro-inc/prod && pulumi preview --diff"
+ },
+ "dependencies": {
+ "@pulumi/datadog": "^4.51.0",
+ "@pulumi/pulumi": "^3.181.0"
+ }
+}
diff --git a/infra/stacks/monitoring/synthetics.ts b/infra/stacks/monitoring/synthetics.ts
new file mode 100644
index 0000000000..9738e72f4f
--- /dev/null
+++ b/infra/stacks/monitoring/synthetics.ts
@@ -0,0 +1,104 @@
+import * as datadog from '@pulumi/datadog';
+
+// The only external probe of the product. It owns its own alert monitor
+// (19726068, "Macro App is Down") through `optionsList.monitorName` — that
+// monitor is not a separate resource and must never be imported as one.
+//
+// `optionsList.monitorOptions` in the provider only models
+// notificationPresetName / escalationMessage / renotify*. The live test also
+// carries onMissingData, notifyAudit, newHostDelay and includeTags on its
+// monitor; those are Datadog's synthetics defaults and are not expressible
+// here.
+//
+// `message` below is Datadog's default synthetics notification template,
+// reproduced verbatim from the live test. Backticks are escaped for the
+// template literal; the rendered text is unchanged.
+export const appWelcomeHealthCheck = new datadog.SyntheticsTest(
+ 'app-welcome-health-check',
+ {
+ name: 'Health check on macro.com/app/welcome',
+ type: 'api',
+ subtype: 'http',
+ status: 'live',
+ tags: ['prod', 'env:prod'],
+ locations: ['aws:ca-central-1', 'aws:us-east-1', 'gcp:us-west2'],
+ requestDefinition: {
+ method: 'GET',
+ url: 'https://macro.com/app/welcome',
+ },
+ assertions: [
+ { type: 'responseTime', operator: 'lessThan', target: '3000' },
+ { type: 'statusCode', operator: 'is', target: '200' },
+ {
+ type: 'header',
+ property: 'content-type',
+ operator: 'is',
+ target: 'text/html',
+ },
+ ],
+ optionsList: {
+ tickEvery: 300,
+ minFailureDuration: 300,
+ minLocationFailed: 1,
+ httpVersion: 'any',
+ retry: { count: 0, interval: 300 },
+ monitorName: 'Macro App is Down',
+ monitorOptions: { notificationPresetName: 'show_all' },
+ },
+ message: `{{! Test result details }}
+
+@slack-holy-shit-alarms @[email protected] @[email protected]
+
+Your test {{#is_alert}}failed{{else}}recovered{{/is_alert}} after running for {{eval "synthetics.attributes.result.duration/1000" }}s on the {{#if synthetics.attributes.location.privateLocation}}Private{{else}}Managed{{/if}} Location {{synthetics.attributes.location.id}}.
+
+{{! If alert, provide details about the failure }}
+{{#is_alert}}{{#is_exact_match "synthetics.attributes.result.failure.code" "INCORRECT_ASSERTION"}}
+{{! Test failed due to failed assertions, list them }}
+# Failed assertions
+{{#each synthetics.attributes.result.assertions}}{{#unless valid}}
+* Assertion \`{{type}}\` \`{{operator}}\`: expected \`{{{expected}}}\`, found \`{{{actual}}}\`
+{{/unless}}{{/each}}
+{{else}}
+{{! Test failed for another reason, show the error }}
+# Error message
+The test failed with the error \`{{{synthetics.attributes.result.failure.message}}}\` (\`{{{synthetics.attributes.result.failure.code}}}\`).
+{{/is_exact_match}}{{/is_alert}}
+
+{{! HTTP request and response details }}
+{{#with synthetics.attributes.result}}
+We made a \`{{request.method}}\` request to {{#if request.service}}the \`{{request.service}}\` service located at {{/if}}\`{{{request.url}}}\` and received an HTTP {{response.httpVersion}} response with a {{response.statusCode}} status code.
+{{/with}}
+
+{{! HTTP Timings }}
+{{#with synthetics.attributes.result.timings}}
+# Timings
+{{#if redirect}}* Redirect: {{redirect}}
+{{/if}}* DNS: {{dns}}
+* TCP: {{tcp}}
+{{#if authentication}}* Authentication: {{authentication}}
+{{/if}}{{#if ssl}}* SSL: {{ssl}}
+{{/if}}* FirstByte: {{firstByte}}
+* Download: {{download}}
+{{/with}}
+
+{{! Display config variables if any }}
+{{#if synthetics.attributes.result.variables.config}}
+# Config Variables
+The test used the following variables:
+{{#each synthetics.attributes.result.variables.config}}
+* **Name:** \`{{name}}\`
+Type: \`{{type}}\`
+Value: {{#if secure}}*Obfuscated (value hidden)*{{else}}\`{{{value}}}\`{{/if}}{{/each}}
+{{/if}}
+
+{{! Display extracted variables, available only if the test is successful }}
+{{#if synthetics.attributes.result.variables.extracted}}
+# Extracted Variables
+The test successfully extracted the following variables:
+{{#each synthetics.attributes.result.variables.extracted}}
+* **Name:** \`{{name}}\` (Global Variable ID: \`{{id}}\`)
+Value: {{#if secure}}*Obfuscated (value hidden)*{{else}}\`{{{val}}}\`{{/if}}{{/each}}
+{{/if}}`,
+ },
+ { protect: true }
+);
diff --git a/infra/stacks/monitoring/tsconfig.json b/infra/stacks/monitoring/tsconfig.json
new file mode 100644
index 0000000000..6f83eb665f
--- /dev/null
+++ b/infra/stacks/monitoring/tsconfig.json
@@ -0,0 +1,3 @@
+{
+ "extends": "../../tsconfig.json",
+}