Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(infra): declare every Datadog monitor in code (#6065)
Wolf Mermelstein60284505495e
diff --git a/crates/documents/src/outbound/editing_worker_client.rs b/crates/documents/src/outbound/editing_worker_client.rsindex 330178feeb..3f1d5d4fb4 100644--- a/crates/documents/src/outbound/editing_worker_client.rs+++ b/crates/documents/src/outbound/editing_worker_client.rs@@ -3,10 +3,14 @@use crate::domain::ports::editing::{EditMode, EditResult, EditUsage, EditingWorkerService, EditorName,};+use anyhow::Context;use macro_sync_service_jwt::DocumentPermissionToken;use reqwest::Client;use std::sync::Arc;+#[cfg(test)]+mod test;+/// Reqwest-backed client for the AI editing worker.#[derive(Clone)]pub struct ReqwestEditingWorkerClient {@@ -230,7 +234,8 @@ impl EditingWorkerService for ReqwestEditingWorkerClient {.headers(headers).json(&request_body).send()- .await?;+ .await+ .context("editing worker request failed")?;let status = edit_resp.status();if !status.is_success() {diff --git a/crates/documents/src/outbound/editing_worker_client/test.rs b/crates/documents/src/outbound/editing_worker_client/test.rsnew file mode 100644index 0000000000..ff6c38961a--- /dev/null+++ b/crates/documents/src/outbound/editing_worker_client/test.rs@@ -0,0 +1,65 @@+use super::*;+use std::time::Duration;++#[tokio::test]+async fn connection_failure_has_monitorable_context_and_preserves_cause() {+ // Reserve a port without listening so connection attempts are refused.+ let socket = tokio::net::TcpSocket::new_v4().unwrap();+ socket.bind("127.0.0.1:0".parse().unwrap()).unwrap();+ let client = ReqwestEditingWorkerClient::new(+ format!("http://{}", socket.local_addr().unwrap()),+ Arc::new(+ Client::builder()+ .no_proxy()+ .timeout(Duration::from_secs(5))+ .build()+ .unwrap(),+ ),+ );++ let error = client+ .edit(+ "test-document",+ &"test-token".to_owned().into(),+ "edit",+ EditMode::Supervised,+ None,+ )+ .await+ .err()+ .expect("a connection to a port that is not listening must fail");++ assert_eq!(error.to_string(), "editing worker request failed");+ assert!(error.downcast_ref::<reqwest::Error>().unwrap().is_connect());+}++#[tokio::test]+async fn timeout_has_monitorable_context_and_preserves_cause() {+ // Keep the connection queued without serving a response.+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let client = ReqwestEditingWorkerClient::new(+ format!("http://{}", listener.local_addr().unwrap()),+ Arc::new(+ Client::builder()+ .no_proxy()+ .timeout(Duration::from_millis(100))+ .build()+ .unwrap(),+ ),+ );++ let error = client+ .edit(+ "test-document",+ &"test-token".to_owned().into(),+ "edit",+ EditMode::Supervised,+ None,+ )+ .await+ .err()+ .expect("a worker that never responds must time out");++ assert_eq!(error.to_string(), "editing worker request failed");+ assert!(error.downcast_ref::<reqwest::Error>().unwrap().is_timeout());+}diff --git a/infra/bun.lock b/infra/bun.lockindex a852f1f779..2c2c1beb34 100644--- a/infra/bun.lock+++ b/infra/bun.lock@@ -171,6 +171,14 @@"name": "mcp-server-stack","version": "0.0.0",},+ "stacks/monitoring": {+ "name": "monitoring-stack",+ "version": "0.0.0",+ "dependencies": {+ "@pulumi/datadog": "^4.51.0",+ "@pulumi/pulumi": "^3.181.0",+ },+ },"stacks/notification-service": {"name": "notification-service-stack","version": "0.0.0",@@ -695,6 +703,8 @@"module-details-from-path": ["[email protected]", "", {}, "sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w=="],+ "monitoring-stack": ["monitoring-stack@workspace:stacks/monitoring"],+"ms": ["[email protected]", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],"negotiator": ["[email protected]", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="],diff --git a/infra/stacks/monitoring/.gitignore b/infra/stacks/monitoring/.gitignorenew file mode 100644index 0000000000..9d5bad2c75--- /dev/null+++ b/infra/stacks/monitoring/.gitignore@@ -0,0 +1,2 @@+# Emitted by `pulumi import --out` during adoption; never committed.+imported.generated.tsdiff --git a/infra/stacks/monitoring/Pulumi.prod.yaml b/infra/stacks/monitoring/Pulumi.prod.yamlnew file mode 100644index 0000000000..138f62896b--- /dev/null+++ b/infra/stacks/monitoring/Pulumi.prod.yaml@@ -0,0 +1,4 @@+# Datadog monitors are org-global, so this project uses only the prod stack.+# The provider reads credentials and the Datadog site from DD_API_KEY,+# DD_APP_KEY, and DD_HOST, supplied locally or by the deployment workflow.+config: {}diff --git a/infra/stacks/monitoring/Pulumi.yaml b/infra/stacks/monitoring/Pulumi.yamlnew file mode 100644index 0000000000..e421141759--- /dev/null+++ b/infra/stacks/monitoring/Pulumi.yaml@@ -0,0 +1,3 @@+name: monitoring+runtime: nodejs+description: Datadog monitors and synthetic testsdiff --git a/infra/stacks/monitoring/README.md b/infra/stacks/monitoring/README.mdnew file mode 100644index 0000000000..852793d3a6--- /dev/null+++ b/infra/stacks/monitoring/README.md@@ -0,0 +1,107 @@+# monitoring++Datadog monitors and synthetic tests as code.++36 resources here (35 monitors and one synthetic test) already exist in+Datadog. This stack **adopts** them — it does not recreate them. `import.json`+lists every one by id. The two AI-editing monitors in `monitors/ai-editing.ts`+are new and are meant to be created.++## Read this before deploying++**Run the import before the first `pulumi up`.** The program declares 35+monitors that already exist in Datadog. Until `pulumi import` has told Pulumi+they exist, a deploy would create 35 duplicates — duplicate pages to+`#holy-shit-alarms` included.++Adoption is a one-time local run. `pulumi import` reads Datadog and writes+Pulumi state — it does not create, modify, or pause anything in Datadog. The+Datadog keys are needed by `preview` as well: the provider validates every+monitor against `/api/v1/monitor/validate` while diffing.++```bash+cd infra/stacks/monitoring+export DD_API_KEY=... DD_APP_KEY=... DD_HOST=https://api.us5.datadoghq.com/+pulumi stack select macro-inc/prod --create+pulumi import --file import.json --out imported.generated.ts --protect --yes+pulumi preview --diff+```++`--protect` matches the `adopted()` helper so state and code agree.+`imported.generated.ts` is Pulumi's own rendering of the live monitors — the+ground truth to reconcile against if the preview shows drift. Delete it after;+it must not be committed (index.ts does not import it, and a second copy of+every monitor would double-declare them).++The gate is that preview:++- **35 monitors + 1 synthetic test unchanged** — adoption is exact.+- **2 monitors to create** (`ai-editing-*`) — expected, that's the new alerting.+- **Any update, replace, or delete** — the hand-written code in `monitors/*.ts`+ drifted from the live monitor. Reconcile against `imported.generated.ts`+ before deploying. An `update` here silently changes who gets paged.++Only then deploy: `pulumi up` locally, or **Actions → "deploy pulumi service"**+with `pulumi-service-name: monitoring`, `environment: prod` (that action already+exports the DD keys from repo secrets).++## Why nothing here deploys by accident++`.github/services-config.json` is the allowlist that drives `deploy_on_push.yml`+and `pulumi_preview_pr.yml`. This stack is deliberately absent from it, so no+push and no PR ever deploys it. It moves only by manual dispatch.++Monitors are org-global rather than per-environment — a single monitor covers+both the `-dev` and `-prod` services it queries. So there is one stack, `prod`,+and no `dev` counterpart. (`deploy-pulumi-stack.yml` names the stack+`macro-inc/<environment>`, hence `prod`.)++## Credentials++None to add. `deploy-cloud-storage-pulumi` already exports `DD_API_KEY`,+`DD_APP_KEY`, and `DD_HOST=https://api.us5.datadoghq.com/`, which is all the+Datadog provider reads. Do not set `datadog:validate: "false"` here — a missing+key should fail the deploy rather than silently no-op.++## Where transcription is most likely to be wrong++The code was written from the Datadog API's own JSON, and `tsc` validates every+field against the provider schema, so field *shapes* are right. Values that the+provider models differently from the API are the risk, and these are the spots:++- `monitors/logs.ts` `error-logs-overall` and `monitors/rum.ts`+ `web-app-source-errors-prod` use `formula()` queries with `variables`. The API+ returns one `compute` object per variable; the provider takes a `computes`+ list. The API also carries `additionalProperties.storage: "hot"` on each+ variable, which provider 4.68 cannot express at all.+- `monitors/rds.ts` `rds-connection-anomaly` is an anomaly monitor, so its+ `monitorThresholdWindows` are required rather than optional.+- `synthetics.ts` reproduces Datadog's default notification template verbatim,+ with backticks escaped for the template literal. Drift there changes alert+ *text*, not alert behavior.+- `onMissingData` values are the API's snake_case (`show_no_data`). Pulumi's+ generated docs camelCase them; the provider forwards the string as-is.++## Drift++A code-managed monitor edited in the UI is reverted on the next deploy of this+stack. Since that deploy is manual, drift shows up in `pulumi preview` before it+is applied. Reach for `restrictedRoles` if convention stops being enough.++## Adding a monitor++Declare it in the matching `monitors/*.ts` with `new datadog.Monitor(...)`. Use+the `adopted()` helper only for monitors that already exist in Datadog — it sets+`protect`, which is there to stop a program that no longer declares a+pre-existing monitor from deleting it.++## AI-editing coverage++The caller-side log alert matches `editing worker returned` for non-2xx HTTP+responses and `editing worker request failed` for transport failures. The latter+requires the updated `ReqwestEditingWorkerClient` to be deployed in the caller+services; creating the monitor alone does not add that telemetry.++Neither new alert detects successful responses that apply no edits, or failures+during periods without editing requests. The end-to-end canary remains a+follow-up.diff --git a/infra/stacks/monitoring/adopted.ts b/infra/stacks/monitoring/adopted.tsnew file mode 100644index 0000000000..5841e6a734--- /dev/null+++ b/infra/stacks/monitoring/adopted.ts@@ -0,0 +1,17 @@+import * as datadog from '@pulumi/datadog';++/**+ * A monitor that already exists in Datadog and was adopted into this stack by+ * `pulumi import` (see README). `protect` is the point: these monitors predate+ * the stack, several of them page on-call, and a program that no longer declares+ * one would otherwise delete it on the next deploy. Protected resources fail the+ * deploy instead.+ *+ * To retire a monitor, drop `protect` in its own commit, then delete it.+ */+export function adopted(+ name: string,+ args: datadog.MonitorArgs+): datadog.Monitor {+ return new datadog.Monitor(name, args, { protect: true });+}diff --git a/infra/stacks/monitoring/import.json b/infra/stacks/monitoring/import.jsonnew file mode 100644index 0000000000..d2577b4933--- /dev/null+++ b/infra/stacks/monitoring/import.json@@ -0,0 +1,40 @@+{+ "resources": [+ { "type": "datadog:index/monitor:Monitor", "name": "macroai-error-logs", "id": "440085" },+ { "type": "datadog:index/monitor:Monitor", "name": "macroapi-error-logs", "id": "440088" },+ { "type": "datadog:index/monitor:Monitor", "name": "dss-errors-dev", "id": "1013923" },+ { "type": "datadog:index/monitor:Monitor", "name": "dss-errors-prod", "id": "1062128" },+ { "type": "datadog:index/monitor:Monitor", "name": "macro-cache-available-memory-low", "id": "1062487" },+ { "type": "datadog:index/monitor:Monitor", "name": "alb-non-200-responses", "id": "1062605" },+ { "type": "datadog:index/monitor:Monitor", "name": "rds-high-cpu", "id": "1062630" },+ { "type": "datadog:index/monitor:Monitor", "name": "macrodb-low-burst-balance", "id": "1062636" },+ { "type": "datadog:index/monitor:Monitor", "name": "cloud-storage-cache-high-cpu", "id": "1062685" },+ { "type": "datadog:index/monitor:Monitor", "name": "cloud-storage-cache-low-memory", "id": "1062687" },+ { "type": "datadog:index/monitor:Monitor", "name": "ecs-high-cpu", "id": "1062688" },+ { "type": "datadog:index/monitor:Monitor", "name": "lambda-error-count", "id": "1065393" },+ { "type": "datadog:index/monitor:Monitor", "name": "error-logs-overall", "id": "1067730" },+ { "type": "datadog:index/monitor:Monitor", "name": "rds-proxy-connections", "id": "1076524" },+ { "type": "datadog:index/monitor:Monitor", "name": "graphql-redis-connect", "id": "1082626" },+ { "type": "datadog:index/monitor:Monitor", "name": "web-app-errors-staging", "id": "1177082" },+ { "type": "datadog:index/monitor:Monitor", "name": "web-app-source-errors-prod", "id": "1189349" },+ { "type": "datadog:index/monitor:Monitor", "name": "document-text-extractor-prod", "id": "1291765" },+ { "type": "datadog:index/monitor:Monitor", "name": "document-text-extractor-dev", "id": "1291772" },+ { "type": "datadog:index/monitor:Monitor", "name": "high-channel-invites", "id": "1333499" },+ { "type": "datadog:index/monitor:Monitor", "name": "ses-bounce-rate", "id": "1333601" },+ { "type": "datadog:index/monitor:Monitor", "name": "ses-complaint-rate", "id": "4178105" },+ { "type": "datadog:index/monitor:Monitor", "name": "ses-high-email-sends", "id": "4757927" },+ { "type": "datadog:index/monitor:Monitor", "name": "channel-invite-spam-dev", "id": "4876245" },+ { "type": "datadog:index/monitor:Monitor", "name": "channel-invite-spam-prod", "id": "4876618" },+ { "type": "datadog:index/monitor:Monitor", "name": "rds-storage-utilization", "id": "6120132" },+ { "type": "datadog:index/monitor:Monitor", "name": "rds-connection-anomaly", "id": "6120237" },+ { "type": "datadog:index/monitor:Monitor", "name": "alb-low-healthy-hosts", "id": "16891631" },+ { "type": "datadog:index/monitor:Monitor", "name": "alb-high-response-time", "id": "16891718" },+ { "type": "datadog:index/monitor:Monitor", "name": "dynamodb-write-capacity", "id": "16917541" },+ { "type": "datadog:index/monitor:Monitor", "name": "dynamodb-read-capacity", "id": "16917574" },+ { "type": "datadog:index/monitor:Monitor", "name": "rds-low-freeable-memory", "id": "16918778" },+ { "type": "datadog:index/monitor:Monitor", "name": "rds-high-iops", "id": "16918920" },+ { "type": "datadog:index/monitor:Monitor", "name": "apm-get-channel-messages-p95", "id": "19271629" },+ { "type": "datadog:index/monitor:Monitor", "name": "opensearch-storage-low", "id": "21836655" },+ { "type": "datadog:index/syntheticsTest:SyntheticsTest", "name": "app-welcome-health-check", "id": "3xv-xcg-7kp" }+ ]+}diff --git a/infra/stacks/monitoring/index.ts b/infra/stacks/monitoring/index.tsnew file mode 100644index 0000000000..4f5f25dc38--- /dev/null+++ b/infra/stacks/monitoring/index.ts@@ -0,0 +1,21 @@+// Datadog monitors and synthetic tests.+//+// Every monitor in the org is declared here. See README.md before deploying —+// the 35 monitors under `monitors/` are adopted, not created, and a program that+// stops declaring one deletes it.++// Adopted from the Datadog UI.+import './monitors/apm';+import './monitors/cache';+import './monitors/compute';+import './monitors/dynamodb';+import './monitors/load-balancer';+import './monitors/logs';+import './monitors/opensearch';+import './monitors/rds';+import './monitors/rum';+import './monitors/ses';+import './synthetics';++// Declared here first.+import './monitors/ai-editing';diff --git a/infra/stacks/monitoring/monitors/ai-editing.ts b/infra/stacks/monitoring/monitors/ai-editing.tsnew file mode 100644index 0000000000..c32f166e5e--- /dev/null+++ b/infra/stacks/monitoring/monitors/ai-editing.ts@@ -0,0 +1,86 @@+// AI document editing.+//+// New monitors, not adopted from the UI, so they are not `protect`ed — they can+// be tuned or removed from code freely.+//+// Two independent angles, because each misses what the other catches. The APM+// monitor sees the worker's own view of its requests; the log monitor sees+// the callers' view, so it still fires when the worker is unreachable and+// reporting no telemetry at all.+//+// Neither covers "the worker answers 200 but silently produces no edits", and+// both are blind while nobody is editing. That needs the hourly canary against+// a fixed document, which is a follow-up: POST /edit requires a short-lived+// `documentToken` (macro_sync_service_jwt), so a probe needs somewhere to mint+// one.+import * as datadog from '@pulumi/datadog';++// The worker's entry span is `http.server.request` with resource_name `POST`.+// Custom spans like `edit.session` do NOT produce trace metrics — Datadog only+// generates them for entry spans, so a monitor on `trace.Internal{...}` here+// would never fire.+export const aiEditingWorkerErrors = new datadog.Monitor(+ 'ai-editing-worker-errors',+ {+ name: '[PROD] AI editing worker returning errors',+ type: 'query alert',+ query:+ 'sum(last_1h):(sum:trace.http.server.request.errors{service:ai-editing-worker,env:production}.as_count() / sum:trace.http.server.request.hits{service:ai-editing-worker,env:production}.as_count()) * 100 > 50',+ message: `More than {{threshold}}% of AI editing requests are failing.++The worker answers 502 when an edit session throws and 499 when the client+cancels; only the former counts as an error here.++- [Worker traces](https://us5.datadoghq.com/apm/traces?query=service%3Aai-editing-worker%20env%3Aproduction%20status%3Aerror)+- Check provider outages first: the supervisor chain falls back Anthropic ->+ OpenAI, so a single provider failing shows up as latency, not errors.++@slack-monitoring`,+ tags: ['env:prod', 'service:ai-editing-worker', 'product:apm'],+ monitorThresholds: { critical: '50', warning: '20' },+ // Ratio of two count metrics: no traffic means no data, which must not+ // alert. Quiet-hours coverage is the canary's job, not this monitor's.+ onMissingData: 'default',+ evaluationDelay: 60,+ includeTags: true,+ notifyAudit: false,+ }+);++// ReqwestEditingWorkerClient::edit reports "editing worker returned" for+// non-2xx responses and "editing worker request failed" for transport failures+// (crates/documents/src/outbound/editing_worker_client.rs).+//+// The callers are document-cognition-service (the EditDocument tool, 174 of+// these in the 30 days to 2026-08-31) and cloud-storage-service (7). Note the+// service tags carry no `-prod` suffix and the status is `warn`, unlike the+// older log monitors in logs.ts — matching on `service:*-prod` here finds+// nothing. agent-schedule-service is excluded: its hits are agent transcripts+// that quote the error text rather than reports of it.+//+// Thresholds come from that 30-day window: the worst normal hour was 12, so+// 25/hour is roughly double the known-bad baseline rather than a guess.+export const aiEditingRequestsFailing = new datadog.Monitor(+ 'ai-editing-requests-failing',+ {+ name: '[PROD] AI editing requests failing for callers',+ type: 'log alert',+ query:+ 'logs("env:prod service:(document-cognition-service OR cloud-storage-service) (\\"editing worker returned\\" OR \\"editing worker request failed\\")").index("*").rollup("count").last("1h") > 25',+ message: `Callers are getting errors back from the AI editing worker.++Unlike the APM monitor this fires even when the worker is unreachable, since it+reads the callers' logs rather than the worker's telemetry. The common cause+historically is \`initial sync failed: timeout\` — the worker could not open the+document over the sync service.++@slack-monitoring`,+ tags: ['env:prod', 'service:ai-editing-worker'],+ monitorThresholds: { critical: '25', warning: '12' },+ onMissingData: 'default',+ includeTags: false,+ notifyAudit: false,+ enableLogsSample: true,+ groupbySimpleMonitor: false,+ }+);diff --git a/infra/stacks/monitoring/monitors/apm.ts b/infra/stacks/monitoring/monitors/apm.tsnew file mode 100644index 0000000000..78577ffcb6--- /dev/null+++ b/infra/stacks/monitoring/monitors/apm.ts@@ -0,0 +1,49 @@+// APM latency.+//+// Adopted from monitors created by hand in the Datadog UI. Do not edit a+// monitor here and in the UI at the same time — this file wins on deploy.+import { adopted } from '../adopted';++// https://us5.datadoghq.com/monitors/19271629+adopted('apm-get-channel-messages-p95', {+ name: 'High p95 Latency – get_channel_messages (cloud-storage-service prod)',+ type: 'query alert',+ query:+ 'percentile(last_15m):p95:trace.Internal{service:cloud-storage-service,resource_name:get_channel_messages,env:prod} > 0.1',+ message: `## ⚠️ High p95 Latency on get_channel_messages++**Service:** cloud-storage-service+**Resource:** get_channel_messages+**Environment:** prod++### What's happening?+p95 response time has exceeded **100ms** (threshold). During the Apr 17 incident this reached 322ms.++### What to check?+1. Check for recent ECS deployments on \`cloud-storage-service\` — the Apr 17 spike was caused by a bad task version+2. Look for \`idle_ns >> busy_ns\` in slow spans — indicates DB wait / connection pool exhaustion+3. Check channels \`0195ceb6-ec2e-7023-80e4-6e084fa6cccd\` and \`0195cea4-5491-77f9-9224-1291fdb8f150\` (repeatedly affected in the last incident)+4. Review the \`Sort(CreatedAt)\` query path on backward pagination (\`direction: Older\`)++### Useful Links+- [Slow spans explorer](https://us5.datadoghq.com/apm/traces?query=service%3Acloud-storage-service+resource_name%3Aget_channel_messages+env%3Aprod+%40duration%3A%3E100000000)+- [Investigation notebook](https://us5.datadoghq.com/notebook/277323)`,+ priority: '5',+ tags: [+ 'service:cloud-storage-service',+ 'env:prod',+ 'resource:get_channel_messages',+ 'product:apm',+ 'team:hutch',+ ],+ monitorThresholds: {+ critical: '0.1',+ warning: '0.05',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: true,+ newHostDelay: 300,+ evaluationDelay: 60,+ requireFullWindow: false,+});diff --git a/infra/stacks/monitoring/monitors/cache.ts b/infra/stacks/monitoring/monitors/cache.tsnew file mode 100644index 0000000000..c40df50b75--- /dev/null+++ b/infra/stacks/monitoring/monitors/cache.ts@@ -0,0 +1,73 @@+// ElastiCache and MemoryDB.+//+// Adopted from monitors created by hand in the Datadog UI. Do not edit a+// monitor here and in the UI at the same time — this file wins on deploy.+import { adopted } from '../adopted';++// https://us5.datadoghq.com/monitors/1062487+adopted('macro-cache-available-memory-low', {+ name: '[PROD] Macro Cache Available Memory Low',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.elasticache.database_memory_usage_percentage{name:macro-cache-prod*} > 90',+ message: `{{#is_alert}}+@slack-holy-shit-alarms+{{/is_alert}}+{{#is_alert_recovery}}+@slack-holy-shit-alarms+{{/is_alert_recovery}}++{{#is_warning_recovery}}+@slack-monitoring+{{/is_warning_recovery}}+{{#is_warning}}+@slack-monitoring+{{/is_warning}}`,+ monitorThresholds: {+ critical: '90',+ warning: '80',+ },+ notifyNoData: false,+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ evaluationDelay: 900,+});++// https://us5.datadoghq.com/monitors/1062685+adopted('cloud-storage-cache-high-cpu', {+ name: '[PROD] Cloud Storage Cache High CPU Utilization',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.memorydb.cpuutilization{clustername:cloud-storage-cache-memorydb-prod} > 90',+ message:+ '{{#is_alert}} @slack-holy-shit-alarms {{/is_alert}} {{#is_alert_recovery}} @slack-holy-shit-alarms {{/is_alert_recovery}}',+ monitorThresholds: {+ critical: '90',+ warning: '80',+ },+ notifyNoData: false,+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ evaluationDelay: 900,+});++// https://us5.datadoghq.com/monitors/1062687+adopted('cloud-storage-cache-low-memory', {+ name: '[PROD] Cloud Storage Cache Low Available Memory',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.memorydb.database_memory_usage_percentage{*} > 90',+ message:+ '{{#is_alert}} @slack-holy-shit-alarms {{/is_alert}} {{#is_alert_recovery}} @slack-holy-shit-alarms {{/is_alert_recovery}}',+ monitorThresholds: {+ critical: '90',+ warning: '60',+ },+ notifyNoData: false,+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ evaluationDelay: 900,+});diff --git a/infra/stacks/monitoring/monitors/compute.ts b/infra/stacks/monitoring/monitors/compute.tsnew file mode 100644index 0000000000..350e6e6c9b--- /dev/null+++ b/infra/stacks/monitoring/monitors/compute.ts@@ -0,0 +1,87 @@+// ECS and Lambda.+//+// Adopted from monitors created by hand in the Datadog UI. Do not edit a+// monitor here and in the UI at the same time — this file wins on deploy.+import { adopted } from '../adopted';++// https://us5.datadoghq.com/monitors/1062688+adopted('ecs-high-cpu', {+ name: '[PROD] High cpu usage across ECS Clusters',+ type: 'query alert',+ query:+ 'avg(last_5m):sum:ecs.fargate.cpu.percent{(ecs_cluster:macro-cluster-prod OR ecs_cluster:cloud-storage-cluster-prod OR ecs_cluster:macroai-cluster-prod-4cbe721)} by {task_name} > 500',+ message: `{{task_name.name}} is experiencing high cpu usage in ECS++@slack-monitoring`,+ monitorThresholds: {+ critical: '500',+ warning: '400',+ },+ onMissingData: 'show_no_data',+ notifyAudit: false,+ includeTags: false,+ newGroupDelay: 0,+});++// https://us5.datadoghq.com/monitors/1065393+adopted('lambda-error-count', {+ name: '[PROD] Lambda Increased Error Count',+ type: 'query alert',+ query:+ 'sum(last_1h):sum:aws.lambda.errors{(level:prod OR environment:prod ) AND NOT functionname:ws-api-gateway-authorizer-lambda-prod} by {functionname}.as_rate() > 500',+ message: `{{#is_alert}}@slack-holy-shit-alarms {{/is_alert}}+{{#is_alert_recovery}}@slack-holy-shit-alarms{{/is_alert_recovery}}++@slack-monitoring`,+ monitorThresholds: {+ critical: '500',+ warning: '100',+ },+ onMissingData: 'show_no_data',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 60,+ evaluationDelay: 900,+});++// https://us5.datadoghq.com/monitors/1291765+adopted('document-text-extractor-prod', {+ name: '[PROD] Document Text Extractor Failing',+ type: 'query alert',+ query:+ 'sum(last_4h):(sum:aws.lambda.errors{resource:document-text-extractor-prod}.as_count() / sum:aws.lambda.invocations{resource:document-text-extractor-prod}.as_count()) * 100 > 80',+ message: `@[email protected]+@slack-monitoring+Document text extractor failing`,+ monitorThresholds: {+ critical: '80',+ warning: '50',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ evaluationDelay: 900,+ renotifyInterval: 0,+});++// https://us5.datadoghq.com/monitors/1291772+adopted('document-text-extractor-dev', {+ name: '[DEV] Document Text Extractor Failing',+ type: 'query alert',+ query:+ 'sum(last_4h):(sum:aws.lambda.errors{resource:document-text-extractor-dev}.as_count() / sum:aws.lambda.invocations{resource:document-text-extractor-dev}.as_count()) * 100 > 80',+ message: `@[email protected]+@slack-monitoring+Document text extractor failing`,+ monitorThresholds: {+ critical: '80',+ warning: '50',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ evaluationDelay: 900,+ renotifyInterval: 0,+});diff --git a/infra/stacks/monitoring/monitors/dynamodb.ts b/infra/stacks/monitoring/monitors/dynamodb.tsnew file mode 100644index 0000000000..067f457696--- /dev/null+++ b/infra/stacks/monitoring/monitors/dynamodb.ts@@ -0,0 +1,43 @@+// DynamoDB consumed capacity.+//+// Adopted from monitors created by hand in the Datadog UI. Do not edit a+// monitor here and in the UI at the same time — this file wins on deploy.+import { adopted } from '../adopted';++// https://us5.datadoghq.com/monitors/16917541+adopted('dynamodb-write-capacity', {+ name: 'Dynamodb ConsumedWriteCapacityUnits High for {{tablename.name}}',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.dynamodb.consumed_write_capacity_units{*} by {tablename} > 500',+ message: `@[email protected]+ monitorThresholds: {+ critical: '500',+ warning: '300',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 60,+ evaluationDelay: 900,+});++// https://us5.datadoghq.com/monitors/16917574+adopted('dynamodb-read-capacity', {+ name: 'Dynamodb ConsumedReadCapacityUnits High for {{tablename.name}}',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.dynamodb.consumed_read_capacity_units{*} by {tablename} > 20',+ message: `@[email protected]+ monitorThresholds: {+ critical: '20',+ warning: '10',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 60,+ evaluationDelay: 900,+});diff --git a/infra/stacks/monitoring/monitors/load-balancer.ts b/infra/stacks/monitoring/monitors/load-balancer.tsnew file mode 100644index 0000000000..dddff183ed--- /dev/null+++ b/infra/stacks/monitoring/monitors/load-balancer.ts@@ -0,0 +1,80 @@+// Application load balancers.+//+// Adopted from monitors created by hand in the Datadog UI. Do not edit a+// monitor here and in the UI at the same time — this file wins on deploy.+import { adopted } from '../adopted';++// https://us5.datadoghq.com/monitors/1062605+adopted('alb-non-200-responses', {+ name: '[PROD] Load Balancer Non-200 Responses',+ type: 'query alert',+ query:+ 'sum(last_1h):(sum:aws.applicationelb.httpcode_elb_5xx{level:prod OR environment:prod*} by {loadbalancer}.as_count() / sum:aws.applicationelb.request_count{level:prod OR environment:prod*} by {loadbalancer}.as_count()) * 100 > 90',+ message: `Detected a high number of non-200 responses for ALB {{loadbalancer.name}}+++{{#is_alert}}+@slack-holy-shit-alarms+{{/is_alert}}+{{#is_alert_recovery}}+@slack-holy-shit-alarms+{{/is_alert_recovery}}++@slack-monitoring+++@slack-monitoring`,+ monitorThresholds: {+ critical: '90',+ warning: '60',+ },+ onMissingData: 'show_no_data',+ notifyAudit: false,+ includeTags: false,+ newGroupDelay: 0,+ evaluationDelay: 900,+ renotifyInterval: 0,+ requireFullWindow: false,+});++// https://us5.datadoghq.com/monitors/16891631+adopted('alb-low-healthy-hosts', {+ name: 'Application load balancer low healthy host count',+ type: 'query alert',+ query:+ 'avg(last_1h):abs(avg:aws.applicationelb.healthy_host_count{*} by {loadbalancer}) <= 0.25',+ message: `The application load balancer for {{loadbalancer.name}} has low healthy host count.+++ monitorThresholds: {+ critical: '0.25',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 60,+ evaluationDelay: 900,+});++// https://us5.datadoghq.com/monitors/16891718+adopted('alb-high-response-time', {+ name: 'High average response time for alb {{loadbalancer.name}}',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.applicationelb.target_response_time.average{! name:proxy-service-alb-prod-153de67 , ! name:store-service-alb-prod-4db6301} by {loadbalancer} > 3',+ message: `Loadbalancer {{loadbalancer.name}} has a high average response time++ monitorThresholds: {+ critical: '3',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 60,+ evaluationDelay: 900,+});diff --git a/infra/stacks/monitoring/monitors/logs.ts b/infra/stacks/monitoring/monitors/logs.tsnew file mode 100644index 0000000000..dac8a31710--- /dev/null+++ b/infra/stacks/monitoring/monitors/logs.ts@@ -0,0 +1,235 @@+// Log-based error-rate monitors.+//+// Adopted from monitors created by hand in the Datadog UI. Do not edit a+// monitor here and in the UI at the same time — this file wins on deploy.+import { adopted } from '../adopted';++// https://us5.datadoghq.com/monitors/440085+adopted('macroai-error-logs', {+ name: '[MACROAI] Increase in Error logs',+ type: 'log alert',+ query:+ 'logs("service:(macroai-query-service-prod OR macroai-store-service-prod OR macroai-proxy-service-prod) status:error").index("*").rollup("count").last("5m") > 2500',+ message: `@[email protected]+@slack-monitoring`,+ tags: ['macro-ai'],+ monitorThresholds: {+ critical: '2500',+ warning: '1000',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ renotifyInterval: 0,+ enableLogsSample: true,+ groupbySimpleMonitor: false,+});++// https://us5.datadoghq.com/monitors/440088+adopted('macroapi-error-logs', {+ name: '[MACROAPI] Increase in Error Logs',+ type: 'log alert',+ query:+ 'logs("service:macro-gql-service-prod status:error").index("*").rollup("count").last("5m") > 50',+ message: '@slack-monitoring',+ monitorThresholds: {+ critical: '50',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ enableLogsSample: false,+ groupbySimpleMonitor: false,+});++// https://us5.datadoghq.com/monitors/1013923+adopted('dss-errors-dev', {+ name: '[DEV] Increased number of errors for DSS',+ type: 'log alert',+ query:+ 'logs("service:cloud-storage-service-dev ERROR*").index("*").rollup("count").last("5m") > 200',+ message: '@slack-monitoring',+ monitorThresholds: {+ critical: '200',+ warning: '100',+ },+ onMissingData: 'default',+ notifyAudit: true,+ includeTags: false,+ newHostDelay: 300,+ enableLogsSample: true,+ groupbySimpleMonitor: false,+});++// https://us5.datadoghq.com/monitors/1062128+adopted('dss-errors-prod', {+ name: '[PROD] Increased number of errors for DSS',+ type: 'log alert',+ query:+ 'logs("service:cloud-storage-service-prod ERROR*").index("*").rollup("count").last("5m") > 150',+ message: '@slack-monitoring',+ monitorThresholds: {+ critical: '150',+ warning: '50',+ },+ onMissingData: 'default',+ notifyAudit: true,+ includeTags: false,+ newHostDelay: 300,+ enableLogsSample: true,+ groupbySimpleMonitor: false,+});++// https://us5.datadoghq.com/monitors/1067730+adopted('error-logs-overall', {+ name: '[PROD] Increase in Error Logs',+ type: 'log alert',+ query: 'formula("query - query1").last("5m") > 25000',+ message: `Increase in overall count of error logs across production services.+Please view datadog to checkout what specific items may be causing the issues.++{{#is_alert}}+@slack-holy-shit-alarms+{{/is_alert}}+{{#is_alert_recovery}}+@slack-holy-shit-alarms+{{/is_alert_recovery}}++@slack-monitoring`,+ priority: '1',+ tags: ['prod'],+ monitorThresholds: {+ critical: '25000',+ warning: '10000',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ enableLogsSample: true,+ groupbySimpleMonitor: false,+ variables: {+ eventQueries: [+ {+ name: 'query',+ dataSource: 'logs',+ computes: [+ {+ aggregation: 'count',+ metric: 'count',+ },+ ],+ search: {+ query:+ '(service:*-prod OR (service:document-processor* @env:prod)) error',+ },+ },+ {+ name: 'query1',+ dataSource: 'logs',+ computes: [+ {+ aggregation: 'count',+ },+ ],+ indexes: ['*'],+ search: {+ query: 'service:macroai-proxy-service-prod error',+ },+ },+ ],+ },+});++// https://us5.datadoghq.com/monitors/1082626+adopted('graphql-redis-connect', {+ name: '[MACROAPI] GraphQL could not connect to Redis',+ type: 'log alert',+ query:+ 'logs("service:macro-gql-service-prod status:error \\"Failed to reconnect to Redis cache at URL*\\"").index("*").rollup("count").last("5m") >= 1',+ message: '@slack-monitoring',+ priority: '3',+ tags: ['prod'],+ monitorThresholds: {+ critical: '1',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ enableLogsSample: true,+ groupbySimpleMonitor: false,+});++// https://us5.datadoghq.com/monitors/1333499+adopted('high-channel-invites', {+ name: 'High Channel Invites',+ type: 'log alert',+ query:+ 'logs("service:notification-service-prod @notification_event_type:channel_invite").index("*").rollup("count", "@sender_id").last("1h") > 100',+ message: `@slack-monitoring+High number of channel invites++Investigate whether 1 user is causing this with this log query++\`service:notification-service-prod @notification_event_type:channel_invite\``,+ monitorThresholds: {+ critical: '100',+ warning: '50',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ enableLogsSample: false,+ groupbySimpleMonitor: false,+});++// https://us5.datadoghq.com/monitors/4876245+adopted('channel-invite-spam-dev', {+ name: '[Channel Invites] [DEV] Potential Email Spam on Channel Invites',+ type: 'log alert',+ query:+ 'logs("service:notification-service-dev message:\\"processing message\\" @notification_event_type:channel_invite").index("*").rollup("count").by("@sender_id").last("1h") > 10',+ message: `A user is generating a high number of channel invites. This could mean they are spamming users with invite emails.++@slack-monitoring++ monitorThresholds: {+ critical: '10',+ warning: '5',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newGroupDelay: 0,+ enableLogsSample: false,+ groupbySimpleMonitor: false,+});++// https://us5.datadoghq.com/monitors/4876618+adopted('channel-invite-spam-prod', {+ name: '[Channel Invites] [PROD] Potential Email Spam on Channel Invites',+ type: 'log alert',+ query:+ 'logs("service:notification-service-prod message:\\"processing message\\" @notification_event_type:channel_invite").index("*").rollup("count").by("@sender_id").last("1h") > 10',+ message: `A user is generating a high number of channel invites. This could mean they are spamming users with invite emails.++@slack-monitoring++ monitorThresholds: {+ critical: '10',+ warning: '5',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newGroupDelay: 0,+ enableLogsSample: false,+ groupbySimpleMonitor: false,+ groupRetentionDuration: '1h',+});diff --git a/infra/stacks/monitoring/monitors/opensearch.ts b/infra/stacks/monitoring/monitors/opensearch.tsnew file mode 100644index 0000000000..022cf98aad--- /dev/null+++ b/infra/stacks/monitoring/monitors/opensearch.ts@@ -0,0 +1,58 @@+// OpenSearch cluster.+//+// Adopted from monitors created by hand in the Datadog UI. Do not edit a+// monitor here and in the UI at the same time — this file wins on deploy.+import { adopted } from '../adopted';++// https://us5.datadoghq.com/monitors/21836655+adopted('opensearch-storage-low', {+ name: '[AWS] Opensearch Storage Decreasing',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.es.free_storage_space.minimum{region:us-east-1} by {environment} < 10000',+ message: `{{#is_alert}}+🚨 CRITICAL — Low Free Opensearch Storage++Environment: {{environment}}+Storage utilization: {{value}}%+Critical threshold: {{threshold}}%++Opensearch storage utilization has reached a critical level. Investigate storage growth or increase the allocated storage.++@webhook-macro-alert-hook+{{/is_alert}}++{{#is_warning}}+⚠️ WARNING — Low Free Opensearch Storage++Environment: {{environment}}+Storage utilization: {{value}}%+Warning threshold: {{warn_threshold}}%++Opensearch storage utilization is approaching the critical threshold.++@webhook-macro-alert-hook+{{/is_warning}}++{{#is_recovery}}+✅ RECOVERED — Opensearch Storage Normal++Environment: {{environment}}+Storage utilization: {{value}}%++Opensearch storage utilization has returned to an acceptable level.++@webhook-macro-alert-hook+{{/is_recovery}}++ monitorThresholds: {+ critical: '10000',+ warning: '20000',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 60,+ evaluationDelay: 900,+});diff --git a/infra/stacks/monitoring/monitors/rds.ts b/infra/stacks/monitoring/monitors/rds.tsnew file mode 100644index 0000000000..6e81abc719--- /dev/null+++ b/infra/stacks/monitoring/monitors/rds.ts@@ -0,0 +1,192 @@+// MacroDB / RDS health.+//+// Adopted from monitors created by hand in the Datadog UI. Do not edit a+// monitor here and in the UI at the same time — this file wins on deploy.+import { adopted } from '../adopted';++// https://us5.datadoghq.com/monitors/1062630+adopted('rds-high-cpu', {+ name: '[PROD] RDS High CPU Utilization',+ type: 'query alert',+ query:+ 'avg(last_1h):sum:aws.rds.cpuutilization{name:*-prod} by {name}.weighted() >= 95',+ message: `@[email protected]+@webhook-macro-alert-hook++RDS DB {{name.name}} is experiencing high CPU utilization.`,+ monitorThresholds: {+ critical: '95',+ warning: '80',+ },+ onMissingData: 'show_no_data',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 0,+ evaluationDelay: 900,+ renotifyInterval: 0,+ requireFullWindow: false,+});++// https://us5.datadoghq.com/monitors/1062636+adopted('macrodb-low-burst-balance', {+ name: '[PROD] MacroDB Low Burst Balance',+ type: 'query alert',+ query: 'avg(last_1h):avg:aws.rds.burst_balance{name:macro-db-prod} <= 10',+ message: '@slack-monitoring',+ monitorThresholds: {+ critical: '10',+ warning: '25',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ evaluationDelay: 900,+});++// https://us5.datadoghq.com/monitors/1076524+adopted('rds-proxy-connections', {+ name: '[PROD] Excessive concurrent connections in rds proxy',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.rds.proxy.database_connections{proxyname:macrodb-proxy-prod} > 500',+ message: `We have excessive concurrent connections in the rds proxy. This can lead to slow database calls and potential outage if other services can not secure a connection++@slack-holy-shit-alarms`,+ monitorThresholds: {+ critical: '500',+ },+ notifyNoData: false,+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ evaluationDelay: 900,+});++// https://us5.datadoghq.com/monitors/6120132+adopted('rds-storage-utilization', {+ name: '[AWS] RDS Storage utilization is high',+ type: 'query alert',+ query:+ 'avg(last_1h):100 - ((avg:aws.rds.free_storage_space{*} by {dbinstanceidentifier,engine} / avg:aws.rds.total_storage_space{*} by {dbinstanceidentifier,engine}) * 100) > 90',+ message: `{{#is_alert}}+🚨 CRITICAL — RDS Storage High++Instance: {{dbinstanceidentifier.name}}+Engine: {{engine.name}}+Storage utilization: {{value}}%+Critical threshold: {{threshold}}%++RDS storage utilization has reached a critical level. Investigate storage growth or increase the allocated storage.++@webhook-macro-alert-hook+{{/is_alert}}++{{#is_warning}}+⚠️ WARNING — RDS Storage High++Instance: {{dbinstanceidentifier.name}}+Engine: {{engine.name}}+Storage utilization: {{value}}%+Warning threshold: {{warn_threshold}}%++RDS storage utilization is approaching the critical threshold.++@webhook-macro-alert-hook+{{/is_warning}}++{{#is_recovery}}+✅ RECOVERED — RDS Storage Normal++Instance: {{dbinstanceidentifier.name}}+Engine: {{engine.name}}+Storage utilization: {{value}}%++RDS storage utilization has returned to an acceptable level.++@webhook-macro-alert-hook+{{/is_recovery}}++ tags: ['integration:amazon_rds'],+ monitorThresholds: {+ critical: '90',+ warning: '85',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 60,+ evaluationDelay: 900,+ renotifyInterval: 0,+ requireFullWindow: false,+});++// https://us5.datadoghq.com/monitors/6120237+adopted('rds-connection-anomaly', {+ name: '[AWS] RDS Anomaly in database connections',+ type: 'query alert',+ query:+ "avg(last_1d):anomalies(avg:aws.rds.database_connections{name:*-prod} by {dbinstanceidentifier}, 'basic', 2, direction='above', interval=300, alert_window='last_1h', count_default_zero='true') >= 0.99",+ message: `The number of connections for RDS database {{dbinstanceidentifier.name}} is outside of the typical range.++@slack-monitoring`,+ tags: ['integration:amazon_rds'],+ monitorThresholds: {+ critical: '0.99',+ criticalRecovery: '0',+ },+ monitorThresholdWindows: {+ recoveryWindow: 'last_15m',+ triggerWindow: 'last_1h',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 60,+ evaluationDelay: 900,+ renotifyInterval: 0,+ requireFullWindow: false,+});++// https://us5.datadoghq.com/monitors/16918778+adopted('rds-low-freeable-memory', {+ name: 'Low freeable memory for {{database_instance.name}}',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.rds.freeable_memory{environment:prod OR database_instance:macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com} by {database_instance} < 2000000000',+ message: `@[email protected]+ monitorThresholds: {+ critical: '2000000000',+ warning: '3000000000',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 60,+ evaluationDelay: 900,+});++// https://us5.datadoghq.com/monitors/16918920+adopted('rds-high-iops', {+ name: 'Database high iops for {{database_instance.name}}',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.rds.write_iops{environment:prod OR database_instance:macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com} by {database_instance}.as_rate() + avg:aws.rds.read_iops{environment:prod OR database_instance:macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com} by {database_instance}.as_rate() > 10000',+ message: `@[email protected]++@webhook-alarm_webhook_test`,+ monitorThresholds: {+ critical: '10000',+ warning: '9000',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 60,+ evaluationDelay: 900,+});diff --git a/infra/stacks/monitoring/monitors/rum.ts b/infra/stacks/monitoring/monitors/rum.tsnew file mode 100644index 0000000000..8c8caae944--- /dev/null+++ b/infra/stacks/monitoring/monitors/rum.ts@@ -0,0 +1,84 @@+// Real User Monitoring alerts on the web app.+//+// Adopted from monitors created by hand in the Datadog UI. Do not edit a+// monitor here and in the UI at the same time — this file wins on deploy.+import { adopted } from '../adopted';++// https://us5.datadoghq.com/monitors/1177082+adopted('web-app-errors-staging', {+ name: '[STAGING] Web-App Increased Error Count',+ type: 'rum alert',+ query:+ 'rum("@type:error @application.name:\\"Web App\\" env:production @view.url_host:staging.macro.com").rollup("count").last("5m") > 90',+ message: '@slack-monitoring',+ monitorThresholds: {+ critical: '90',+ warning: '50',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ enableLogsSample: false,+ groupbySimpleMonitor: false,+});++// https://us5.datadoghq.com/monitors/1189349+adopted('web-app-source-errors-prod', {+ name: '[PROD] Web-App Source Error',+ type: 'rum alert',+ query: 'formula("(1 - (query - query1) / query) * 100").last("1h") > 90',+ message: `{{value}} % of all views in Web App are throwing errors from source.++@slack-monitoring+{{#is_alert}}@slack-holy-shit-alarms {{/is_alert}}`,+ priority: '2',+ tags: ['prod'],+ monitorThresholds: {+ critical: '90',+ warning: '50',+ },+ onMissingData: 'show_no_data',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ renotifyInterval: 60,+ renotifyStatuses: ['alert'],+ notificationPresetName: 'hide_all',+ enableLogsSample: false,+ groupbySimpleMonitor: false,+ variables: {+ eventQueries: [+ {+ name: 'query',+ dataSource: 'rum',+ computes: [+ {+ aggregation: 'cardinality',+ metric: '@view.id',+ },+ ],+ indexes: ['*'],+ search: {+ query:+ '@type:view (@application.name:"Web App" OR service:web-app) env:production @view.url_host:(macro.com OR www.macro.com)',+ },+ },+ {+ name: 'query1',+ dataSource: 'rum',+ computes: [+ {+ aggregation: 'cardinality',+ metric: '@view.id',+ },+ ],+ indexes: ['*'],+ search: {+ query:+ '@type:error (@application.name:"Web App" OR service:web-app) env:production @view.url_host:(macro.com OR www.macro.com) @error.source:source',+ },+ },+ ],+ },+});diff --git a/infra/stacks/monitoring/monitors/ses.ts b/infra/stacks/monitoring/monitors/ses.tsnew file mode 100644index 0000000000..ce427ff29b--- /dev/null+++ b/infra/stacks/monitoring/monitors/ses.ts@@ -0,0 +1,78 @@+// SES sending reputation.+//+// Adopted from monitors created by hand in the Datadog UI. Do not edit a+// monitor here and in the UI at the same time — this file wins on deploy.+import { adopted } from '../adopted';++// https://us5.datadoghq.com/monitors/1333601+adopted('ses-bounce-rate', {+ name: '[SES] Bounce Rate Increasing',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.ses.reputation_bounce_rate{aws_account:569036502058} > 0.05',+ message: `SES Bounce Rate Increasing.++{{#is_alert}} @slack-holy-shit-alarms {{/is_alert}} {{#is_alert_recovery}} @slack-holy-shit-alarms {{/is_alert_recovery}}++@slack-monitoring`,+ monitorThresholds: {+ critical: '0.05',+ warning: '0.03',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ evaluationDelay: 900,+});++// https://us5.datadoghq.com/monitors/4178105+adopted('ses-complaint-rate', {+ name: '[SES] Complaint Rate Increasing',+ type: 'query alert',+ query:+ 'avg(last_1h):avg:aws.ses.reputation_complaint_rate{aws_account:569036502058} > 0.005',+ message: `SES Bounce Rate Increasing. Action needs to be taken to prevent account suspension.++{{#is_warning}}@slack-monitoring{{/is_warning}}+{{#is_warning_recovery}}@slack-monitoring {{/is_warning_recovery}}++{{#is_alert}}@slack-holy-shit-alarms {{/is_alert}}+{{#is_alert_recovery}}@slack-holy-shit-alarms @slack-monitoring {{/is_alert_recovery}}++ monitorThresholds: {+ critical: '0.005',+ warning: '0.001',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: false,+ newHostDelay: 300,+ evaluationDelay: 900,+});++// https://us5.datadoghq.com/monitors/4757927+adopted('ses-high-email-sends', {+ name: '[SES] HIGH EMAIL SENDS',+ type: 'query alert',+ query: 'sum(last_1h):avg:aws.ses.send.sum{*} by {aws_account} > 250',+ message: `SES is experiencing high email sends. This could be related to a potential attack and may require users to be blocked.++{{#is_warning}}@slack-monitoring{{/is_warning}}+{{#is_warning_recovery}}@slack-monitoring {{/is_warning_recovery}}++{{#is_alert}}@slack-holy-shit-alarms {{/is_alert}}+{{#is_alert_recovery}}@slack-holy-shit-alarms @slack-monitoring {{/is_alert_recovery}}++ monitorThresholds: {+ critical: '250',+ warning: '200',+ },+ onMissingData: 'default',+ notifyAudit: false,+ includeTags: true,+ newGroupDelay: 60,+ evaluationDelay: 900,+});diff --git a/infra/stacks/monitoring/package.json b/infra/stacks/monitoring/package.jsonnew file mode 100644index 0000000000..43e0c93469--- /dev/null+++ b/infra/stacks/monitoring/package.json@@ -0,0 +1,14 @@+{+ "name": "monitoring-stack",+ "version": "0.0.0",+ "private": true,+ "license": "MIT",+ "main": "index.ts",+ "scripts": {+ "preview:prod": "pulumi stack select macro-inc/prod && pulumi preview --diff"+ },+ "dependencies": {+ "@pulumi/datadog": "^4.51.0",+ "@pulumi/pulumi": "^3.181.0"+ }+}diff --git a/infra/stacks/monitoring/synthetics.ts b/infra/stacks/monitoring/synthetics.tsnew file mode 100644index 0000000000..9738e72f4f--- /dev/null+++ b/infra/stacks/monitoring/synthetics.ts@@ -0,0 +1,104 @@+import * as datadog from '@pulumi/datadog';++// The only external probe of the product. It owns its own alert monitor+// (19726068, "Macro App is Down") through `optionsList.monitorName` — that+// monitor is not a separate resource and must never be imported as one.+//+// `optionsList.monitorOptions` in the provider only models+// notificationPresetName / escalationMessage / renotify*. The live test also+// carries onMissingData, notifyAudit, newHostDelay and includeTags on its+// monitor; those are Datadog's synthetics defaults and are not expressible+// here.+//+// `message` below is Datadog's default synthetics notification template,+// reproduced verbatim from the live test. Backticks are escaped for the+// template literal; the rendered text is unchanged.+export const appWelcomeHealthCheck = new datadog.SyntheticsTest(+ 'app-welcome-health-check',+ {+ name: 'Health check on macro.com/app/welcome',+ type: 'api',+ subtype: 'http',+ status: 'live',+ tags: ['prod', 'env:prod'],+ locations: ['aws:ca-central-1', 'aws:us-east-1', 'gcp:us-west2'],+ requestDefinition: {+ method: 'GET',+ url: 'https://macro.com/app/welcome',+ },+ assertions: [+ { type: 'responseTime', operator: 'lessThan', target: '3000' },+ { type: 'statusCode', operator: 'is', target: '200' },+ {+ type: 'header',+ property: 'content-type',+ operator: 'is',+ target: 'text/html',+ },+ ],+ optionsList: {+ tickEvery: 300,+ minFailureDuration: 300,+ minLocationFailed: 1,+ httpVersion: 'any',+ retry: { count: 0, interval: 300 },+ monitorName: 'Macro App is Down',+ monitorOptions: { notificationPresetName: 'show_all' },+ },+ message: `{{! Test result details }}++@slack-holy-shit-alarms @[email protected] @[email protected]++Your test {{#is_alert}}failed{{else}}recovered{{/is_alert}} after running for {{eval "synthetics.attributes.result.duration/1000" }}s on the {{#if synthetics.attributes.location.privateLocation}}Private{{else}}Managed{{/if}} Location {{synthetics.attributes.location.id}}.++{{! If alert, provide details about the failure }}+{{#is_alert}}{{#is_exact_match "synthetics.attributes.result.failure.code" "INCORRECT_ASSERTION"}}+{{! Test failed due to failed assertions, list them }}+# Failed assertions+{{#each synthetics.attributes.result.assertions}}{{#unless valid}}+* Assertion \`{{type}}\` \`{{operator}}\`: expected \`{{{expected}}}\`, found \`{{{actual}}}\`+{{/unless}}{{/each}}+{{else}}+{{! Test failed for another reason, show the error }}+# Error message+The test failed with the error \`{{{synthetics.attributes.result.failure.message}}}\` (\`{{{synthetics.attributes.result.failure.code}}}\`).+{{/is_exact_match}}{{/is_alert}}++{{! HTTP request and response details }}+{{#with synthetics.attributes.result}}+We made a \`{{request.method}}\` request to {{#if request.service}}the \`{{request.service}}\` service located at {{/if}}\`{{{request.url}}}\` and received an HTTP {{response.httpVersion}} response with a {{response.statusCode}} status code.+{{/with}}++{{! HTTP Timings }}+{{#with synthetics.attributes.result.timings}}+# Timings+{{#if redirect}}* Redirect: {{redirect}}+{{/if}}* DNS: {{dns}}+* TCP: {{tcp}}+{{#if authentication}}* Authentication: {{authentication}}+{{/if}}{{#if ssl}}* SSL: {{ssl}}+{{/if}}* FirstByte: {{firstByte}}+* Download: {{download}}+{{/with}}++{{! Display config variables if any }}+{{#if synthetics.attributes.result.variables.config}}+# Config Variables+The test used the following variables:+{{#each synthetics.attributes.result.variables.config}}+* **Name:** \`{{name}}\`+Type: \`{{type}}\`+Value: {{#if secure}}*Obfuscated (value hidden)*{{else}}\`{{{value}}}\`{{/if}}{{/each}}+{{/if}}++{{! Display extracted variables, available only if the test is successful }}+{{#if synthetics.attributes.result.variables.extracted}}+# Extracted Variables+The test successfully extracted the following variables:+{{#each synthetics.attributes.result.variables.extracted}}+* **Name:** \`{{name}}\` (Global Variable ID: \`{{id}}\`)+Value: {{#if secure}}*Obfuscated (value hidden)*{{else}}\`{{{val}}}\`{{/if}}{{/each}}+{{/if}}`,+ },+ { protect: true }+);diff --git a/infra/stacks/monitoring/tsconfig.json b/infra/stacks/monitoring/tsconfig.jsonnew file mode 100644index 0000000000..6f83eb665f--- /dev/null+++ b/infra/stacks/monitoring/tsconfig.json@@ -0,0 +1,3 @@+{+ "extends": "../../tsconfig.json",+}