Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

7.6 KB
Raw Blame
// Log-based error-rate monitors.
//
// Adopted from monitors created by hand in the Datadog UI. Do not edit a
// monitor here and in the UI at the same time — this file wins on deploy.
import { adopted } from '../adopted';
import { notify } from '../notify';

// DSS logs every expired-JWT rejection at ERROR. These are routine client token
// refreshes and make up about 60% of its prod error lines, so the DSS error
// monitors drop them. Log search wildcards are case-sensitive, so each spelling
// (`JwtExpired`, `jwt is expired`, `jwt expired`) is listed; `?` stands in for
// the space.
const DSS_EXPIRED_JWT_EXCLUSION =
  '-@error:(*JwtExpired* OR *jwt?is?expired* OR *jwt?expired*)';

// https://us5.datadoghq.com/monitors/440085
adopted('macroai-error-logs', {
  name: '[MACROAI] Increase in Error logs',
  type: 'log alert',
  query:
    'logs("service:(macroai-query-service-prod OR macroai-store-service-prod OR macroai-proxy-service-prod) status:error").index("*").rollup("count").last("5m") > 2500',
  message: `macroai services logged {{value}} errors in 5 minutes.

${notify({ pages: false })}`,
  tags: ['macro-ai'],
  monitorThresholds: {
    critical: '2500',
    warning: '1000',
  },
  onMissingData: 'default',
  notifyAudit: false,
  includeTags: false,
  newHostDelay: 300,
  enableLogsSample: true,
  groupbySimpleMonitor: false,
});

// https://us5.datadoghq.com/monitors/440088
adopted('macroapi-error-logs', {
  name: '[MACROAPI] Increase in Error Logs',
  type: 'log alert',
  query:
    'logs("service:macro-gql-service-prod status:error").index("*").rollup("count").last("5m") > 50',
  message: `macro-gql-service logged {{value}} errors in 5 minutes.

${notify({ pages: false })}`,
  monitorThresholds: {
    critical: '50',
  },
  onMissingData: 'default',
  notifyAudit: false,
  includeTags: false,
  newHostDelay: 300,
  enableLogsSample: false,
  groupbySimpleMonitor: false,
});

// https://us5.datadoghq.com/monitors/1013923
adopted('dss-errors-dev', {
  name: '[DEV] Increased number of errors for DSS',
  type: 'log alert',
  query: `logs("service:cloud-storage-service env:dev status:error ${DSS_EXPIRED_JWT_EXCLUSION}").index("*").rollup("count").last("5m") > 200`,
  message: `Dev cloud-storage-service logged {{value}} errors in 5 minutes.

${notify({ pages: false, environment: 'dev' })}`,
  monitorThresholds: {
    critical: '200',
    warning: '100',
  },
  onMissingData: 'default',
  notifyAudit: true,
  includeTags: false,
  newHostDelay: 300,
  enableLogsSample: true,
  groupbySimpleMonitor: false,
});

// https://us5.datadoghq.com/monitors/1062128
// The steady baseline is 300-450 error lines per 5 minutes since the
// expired-JWT exclusion, so 800 is roughly double a normal window. No warning
// threshold: one would notify all day.
adopted('dss-errors-prod', {
  name: '[PROD] Increased number of errors for DSS',
  type: 'log alert',
  query: `logs("service:cloud-storage-service env:prod status:error ${DSS_EXPIRED_JWT_EXCLUSION}").index("*").rollup("count").last("5m") > 800`,
  message: `Prod cloud-storage-service logged {{value}} errors in 5 minutes, not counting expired-JWT rejections.

${notify({ pages: false })}`,
  monitorThresholds: {
    critical: '800',
  },
  onMissingData: 'default',
  notifyAudit: true,
  includeTags: false,
  newHostDelay: 300,
  enableLogsSample: true,
  groupbySimpleMonitor: false,
});

// https://us5.datadoghq.com/monitors/1067730
adopted('error-logs-overall', {
  name: '[PROD] Increase in Error Logs',
  type: 'log alert',
  query: 'formula("query - query1").last("5m") > 25000',
  // Crosses its thresholds many times a day, so it notifies without paging.
  // The per-service anomaly monitors in regressions.ts cover regressions.
  message: `Increase in overall count of error logs across production services.
Please view datadog to checkout what specific items may be causing the issues.

${notify({ pages: false })}`,
  priority: '1',
  tags: ['prod'],
  monitorThresholds: {
    critical: '25000',
    warning: '10000',
  },
  onMissingData: 'default',
  notifyAudit: false,
  includeTags: false,
  newHostDelay: 300,
  enableLogsSample: true,
  groupbySimpleMonitor: false,
  variables: {
    eventQueries: [
      {
        name: 'query',
        dataSource: 'logs',
        computes: [
          {
            aggregation: 'count',
            metric: 'count',
          },
        ],
        search: {
          query:
            '(service:*-prod OR (service:document-processor* @env:prod)) error',
        },
      },
      {
        name: 'query1',
        dataSource: 'logs',
        computes: [
          {
            aggregation: 'count',
          },
        ],
        indexes: ['*'],
        search: {
          query: 'service:macroai-proxy-service-prod error',
        },
      },
    ],
  },
});

// https://us5.datadoghq.com/monitors/1082626
adopted('graphql-redis-connect', {
  name: '[MACROAPI] GraphQL could not connect to Redis',
  type: 'log alert',
  query:
    'logs("service:macro-gql-service-prod status:error \\"Failed to reconnect to Redis cache at URL*\\"").index("*").rollup("count").last("5m") >= 1',
  message: `macro-gql-service could not reconnect to its Redis cache.

${notify({ pages: false })}`,
  priority: '3',
  tags: ['prod'],
  monitorThresholds: {
    critical: '1',
  },
  onMissingData: 'default',
  notifyAudit: false,
  includeTags: false,
  newHostDelay: 300,
  enableLogsSample: true,
  groupbySimpleMonitor: false,
});

// https://us5.datadoghq.com/monitors/1333499
adopted('high-channel-invites', {
  name: 'High Channel Invites',
  type: 'log alert',
  query:
    'logs("service:notification-service-prod @notification_event_type:channel_invite").index("*").rollup("count", "@sender_id").last("1h") > 100',
  message: `High number of channel invites

Investigate whether 1 user is causing this with this log query

\`service:notification-service-prod @notification_event_type:channel_invite\`

${notify({ pages: false })}`,
  monitorThresholds: {
    critical: '100',
    warning: '50',
  },
  onMissingData: 'default',
  notifyAudit: false,
  includeTags: false,
  newHostDelay: 300,
  enableLogsSample: false,
  groupbySimpleMonitor: false,
});

// https://us5.datadoghq.com/monitors/4876245
adopted('channel-invite-spam-dev', {
  name: '[Channel Invites] [DEV] Potential Email Spam on Channel Invites',
  type: 'log alert',
  query:
    'logs("service:notification-service-dev message:\\"processing message\\" @notification_event_type:channel_invite").index("*").rollup("count").by("@sender_id").last("1h") > 10',
  message: `A user is generating a high number of channel invites. This could mean they are spamming users with invite emails.

${notify({ pages: false, environment: 'dev' })}`,
  monitorThresholds: {
    critical: '10',
    warning: '5',
  },
  onMissingData: 'default',
  notifyAudit: false,
  includeTags: false,
  newGroupDelay: 0,
  enableLogsSample: false,
  groupbySimpleMonitor: false,
});

// https://us5.datadoghq.com/monitors/4876618
adopted('channel-invite-spam-prod', {
  name: '[Channel Invites] [PROD] Potential Email Spam on Channel Invites',
  type: 'log alert',
  query:
    'logs("service:notification-service-prod message:\\"processing message\\" @notification_event_type:channel_invite").index("*").rollup("count").by("@sender_id").last("1h") > 10',
  message: `A user is generating a high number of channel invites. This could mean they are spamming users with invite emails.

${notify({ pages: false })}`,
  monitorThresholds: {
    critical: '10',
    warning: '5',
  },
  onMissingData: 'default',
  notifyAudit: false,
  includeTags: false,
  newGroupDelay: 0,
  enableLogsSample: false,
  groupbySimpleMonitor: false,
  groupRetentionDuration: '1h',
});