Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

12.1 KB
Raw Blame
// MacroDB (macro-db-prod and its read replica) health.
//
// Thresholds come from two weeks of 5-minute data before the October overload.
// Monitors on CloudWatch metrics also exist in Grafana
// (infra/stacks/observability/nixos/prod-alerts.nix); change both together.
// Grafana pages on-call for those through GoAlert, so they only notify here.
import * as datadog from '@pulumi/datadog';
import { graphGroup, prodMonitor, thresholdGraph } from '../prod-monitor';

// One graph per monitor below, titled like it, with its thresholds.
const dashboard = new datadog.Dashboard('macrodb-alerts', {
  title: 'MacroDB alerts',
  description:
    'Every prod MacroDB monitor against its thresholds. Declared in infra/stacks/monitoring/monitors/macrodb.ts; see docs/runbooks/macrodb.md.',
  layoutType: 'ordered',
  widgets: [
    graphGroup('Memory', [
      thresholdGraph({
        title: 'macro-db-prod freeable memory low',
        query:
          'avg:aws.rds.freeable_memory{dbinstanceidentifier:macro-db-prod}',
        critical: '1000000000',
        warning: '2000000000',
      }),
      thresholdGraph({
        title: 'macro-db-prod-read-replica freeable memory low',
        query:
          'avg:aws.rds.freeable_memory{dbinstanceidentifier:macro-db-prod-read-replica}',
        critical: '750000000',
        warning: '1500000000',
      }),
      thresholdGraph({
        title: 'macro-db-prod swapping',
        query:
          'max:aws.rds.swap_usage{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier}',
        critical: '5000000000',
        warning: '2000000000',
      }),
    ]),
    graphGroup('Load', [
      thresholdGraph({
        title: 'macro-db-prod database load above vCPU count',
        query:
          'avg:aws.rds.dbload_relative_to_num_vcpus{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier}',
        critical: '4',
        warning: '2',
      }),
      thresholdGraph({
        title: 'macro-db-prod CPU high',
        query:
          'avg:aws.rds.cpuutilization{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier}',
        critical: '85',
        warning: '70',
      }),
      thresholdGraph({
        title: 'macro-db-prod disk read latency high',
        query:
          'avg:aws.rds.read_latency{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier}',
        critical: '0.01',
        warning: '0.005',
      }),
    ]),
    graphGroup('Connections and replication', [
      thresholdGraph({
        title: 'macro-db-prod connections high',
        query:
          'max:aws.rds.database_connections{dbinstanceidentifier:macro-db-prod}',
        critical: '2200',
        warning: '1700',
      }),
      thresholdGraph({
        title: 'macro-db-prod-read-replica connections high',
        query:
          'max:aws.rds.database_connections{dbinstanceidentifier:macro-db-prod-read-replica}',
        critical: '700',
        warning: '550',
      }),
      thresholdGraph({
        title: 'macro-db-prod-read-replica lagging',
        query:
          'max:aws.rds.replica_lag{dbinstanceidentifier:macro-db-prod-read-replica}',
        critical: '300',
        warning: '60',
      }),
    ]),
    graphGroup('Locks', [
      thresholdGraph({
        title: 'macro-db-prod connections blocked on locks',
        query:
          'max:postgresql.activity.blocked_connections{host:macro-db-prod*} by {host}',
        critical: '50',
        warning: '20',
      }),
      thresholdGraph({
        title: 'macro-db-prod long-running transaction',
        query:
          "moving_rollup(max:postgresql.activity.xact_start_age{host:macro-db-prod*} by {host}, 300, 'min')",
        critical: '1500',
        warning: '300',
      }),
    ]),
    graphGroup('Storage', [
      thresholdGraph({
        title: 'macro-db-prod free storage low',
        query:
          'min:aws.rds.free_storage_space{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier}',
        critical: '1000000000000',
        warning: '1500000000000',
      }),
      thresholdGraph({
        title: 'macro-db-prod free storage forecast to run low within a week',
        query:
          "forecast(min:aws.rds.free_storage_space{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier}, 'linear', 1)",
        critical: '500000000000',
      }),
    ]),
  ],
});

prodMonitor('macrodb-primary-freeable-memory', {
  title: 'macro-db-prod freeable memory low',
  service: 'macro-db-prod',
  query:
    'avg(last_15m):avg:aws.rds.freeable_memory{dbinstanceidentifier:macro-db-prod} < 1000000000',
  monitorThresholds: { critical: '1000000000', warning: '2000000000' },
  pages: false,
  description:
    'Freeable memory is {{value}} bytes. Normal is 7-18 GB; it fell to about 0.45 GB during the October overload. Each connection holds about 14 MB of private memory, so check connection count and swap first.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#memory',
});

prodMonitor('macrodb-replica-freeable-memory', {
  title: 'macro-db-prod-read-replica freeable memory low',
  service: 'macro-db-prod',
  query:
    'avg(last_15m):avg:aws.rds.freeable_memory{dbinstanceidentifier:macro-db-prod-read-replica} < 750000000',
  monitorThresholds: { critical: '750000000', warning: '1500000000' },
  pages: false,
  description:
    'Replica freeable memory is {{value}} bytes. Normal minimum is about 3 GB. The replica serves GraphQL reads, so memory pressure shows up as slow lists.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#memory',
});

prodMonitor('macrodb-swap', {
  title: 'macro-db-prod swapping',
  service: 'macro-db-prod',
  query:
    'avg(last_15m):max:aws.rds.swap_usage{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier} > 5000000000',
  monitorThresholds: { critical: '5000000000', warning: '2000000000' },
  pages: false,
  description:
    '{{dbinstanceidentifier.name}} is using {{value}} bytes of swap. Normal is under 0.6 GB. Swap above 5 GB preceded both October overloads by hours.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#memory',
});

prodMonitor('macrodb-load', {
  title: 'macro-db-prod database load above vCPU count',
  service: 'macro-db-prod',
  query:
    'avg(last_15m):avg:aws.rds.dbload_relative_to_num_vcpus{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier} > 4',
  monitorThresholds: { critical: '4', warning: '2' },
  pages: false,
  description:
    '{{dbinstanceidentifier.name}} has {{value}}x as many active sessions as vCPUs. Normal is under 1.3; the October overload reached 8. Look at Performance Insights top waits (locks, IO) and the services holding the most connections.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#load',
});

prodMonitor('macrodb-cpu', {
  title: 'macro-db-prod CPU high',
  service: 'macro-db-prod',
  query:
    'avg(last_15m):avg:aws.rds.cpuutilization{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier} > 85',
  monitorThresholds: { critical: '85', warning: '70' },
  pages: false,
  description:
    '{{dbinstanceidentifier.name}} CPU averaged {{value}}% over 15 minutes. Normal peaks are about 30%.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#load',
});

prodMonitor('macrodb-primary-connections', {
  title: 'macro-db-prod connections high',
  service: 'macro-db-prod',
  query:
    'avg(last_10m):max:aws.rds.database_connections{dbinstanceidentifier:macro-db-prod} > 2200',
  monitorThresholds: { critical: '2200', warning: '1700' },
  pages: false,
  description:
    'macro-db-prod has {{value}} connections. Normal 15-minute peaks are about 1,100; max_connections is 3,439. Each connection costs about 14 MB, so this also drains freeable memory.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#connections',
});

prodMonitor('macrodb-replica-connections', {
  title: 'macro-db-prod-read-replica connections high',
  service: 'macro-db-prod',
  query:
    'avg(last_10m):max:aws.rds.database_connections{dbinstanceidentifier:macro-db-prod-read-replica} > 700',
  monitorThresholds: { critical: '700', warning: '550' },
  pages: false,
  description:
    'The read replica has {{value}} connections. Normal peaks are about 460.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#connections',
});

prodMonitor('macrodb-replica-lag', {
  title: 'macro-db-prod-read-replica lagging',
  service: 'macro-db-prod',
  query:
    'avg(last_10m):max:aws.rds.replica_lag{dbinstanceidentifier:macro-db-prod-read-replica} > 300',
  monitorThresholds: { critical: '300', warning: '60' },
  pages: false,
  description:
    'The read replica is {{value}} seconds behind the primary. Normal is under 6 seconds. GraphQL reads use the replica, so users see stale data and missing new items.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#replica-lag',
});

prodMonitor('macrodb-read-latency', {
  title: 'macro-db-prod disk read latency high',
  service: 'macro-db-prod',
  query:
    'avg(last_15m):avg:aws.rds.read_latency{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier} > 0.01',
  monitorThresholds: { critical: '0.01', warning: '0.005' },
  pages: false,
  description:
    '{{dbinstanceidentifier.name}} disk reads take {{value}} seconds on average. Normal is about 0.6 ms.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#load',
});

prodMonitor('macrodb-free-storage', {
  title: 'macro-db-prod free storage low',
  service: 'macro-db-prod',
  query:
    'avg(last_1h):min:aws.rds.free_storage_space{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier} < 1000000000000',
  monitorThresholds: { critical: '1000000000000', warning: '1500000000000' },
  pages: false,
  description:
    '{{dbinstanceidentifier.name}} has {{value}} bytes of free storage out of 5.37 TB.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#storage',
});

// Datadog only: Grafana alerting has no forecast function.
prodMonitor('macrodb-free-storage-forecast', {
  title: 'macro-db-prod free storage forecast to run low within a week',
  service: 'macro-db-prod',
  query:
    "min(next_1w):forecast(min:aws.rds.free_storage_space{dbinstanceidentifier:macro-db-prod*} by {dbinstanceidentifier}, 'linear', 1, interval='60m', history='1w', model='default') <= 500000000000",
  monitorThresholds: { critical: '500000000000' },
  pages: false,
  description:
    'At the current write rate {{dbinstanceidentifier.name}} drops below 500 GB free within a week. Find what started writing; recent growth was 100-200 GB a day.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#storage',
});

// Datadog only: Database Monitoring metrics; Grafana's Postgres collector runs
// against dev only.
prodMonitor('macrodb-blocked-connections', {
  title: 'macro-db-prod connections blocked on locks',
  service: 'macro-db-prod',
  query:
    'avg(last_10m):max:postgresql.activity.blocked_connections{host:macro-db-prod*} by {host} > 50',
  monitorThresholds: { critical: '50', warning: '20' },
  pages: true,
  description:
    "{{host.name}} has {{value}} connections waiting on locks. Normal is under 7. Advisory and tuple locks dominated the October overload; see Database Monitoring's blocking queries view.",
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#locks',
});

// The lowest age over 5 minutes, so one sample cannot trigger it: an idle
// LISTEN backend draining NOTIFYs reports its connect time as xact_start for
// a few seconds (docs/runbooks/macrodb.md#locks). An open transaction stays
// over 300s for 5 minutes once it is 10 minutes old, so warning and critical
// fire at about 10 and 30 minutes.
prodMonitor('macrodb-long-transactions', {
  title: 'macro-db-prod long-running transaction',
  service: 'macro-db-prod',
  query:
    'min(last_5m):max:postgresql.activity.xact_start_age{host:macro-db-prod*} by {host} > 1500',
  monitorThresholds: { critical: '1500', warning: '300' },
  pages: false,
  description:
    'The oldest open transaction on {{host.name}} has been over {{value}} seconds old for 5 minutes, so it has been open at least 5 minutes longer than that. Long transactions hold locks and stop vacuum; find the session and its SQL in Database Monitoring.',
  dashboard: dashboard.url,
  runbook: 'docs/runbooks/macrodb.md#locks',
});