Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

14.0 KB
Raw Blame
import * as aws from '@pulumi/aws';
import * as pulumi from '@pulumi/pulumi';
import { createCiTelemetryRole } from './ci';
import { createNetwork } from './network';
import { renderUserData } from './render';
import {
  validateRegion,
  validateRegionalArn,
  validateSettings,
} from './settings';

const config = new pulumi.Config();
const stack = pulumi.getStack();
if (!['dev', 'prod'].includes(stack)) {
  throw new Error('Use the dev or prod stack to select a unique hostname');
}
const region = aws.config.requireRegion();
validateRegion(region);
const baseDomain = 'macro-internal.com';
const suffix = stack === 'prod' ? '' : '-dev';
const grafanaHost = `grafana${suffix}.${baseDomain}`;
const otlpHost = `otlp${suffix}.${baseDomain}`;
const oncallHost = `oncall${suffix}.${baseDomain}`;
const tags = { project: 'observability', environment: stack };
const durable = { protect: true, retainOnDelete: true };
const secretArn = config.require('secretArn');
const alarmTopicArn = config.require('alarmTopicArn');
// Built from this stack's pinned NixOS flake and published in Ohio.
const amiId = config.require('amiId');
if (!/^ami-[a-f0-9]+$/.test(amiId)) {
  throw new Error('amiId must identify the reviewed observability NixOS image');
}
// Validate operator input before registering any infrastructure resources.
validateSettings({
  region,
  grafanaHost,
  otlpHost,
  oncallHost,
  secretArn,
  alarmTopicArn,
  volumeId: 'vol-0',
  logsBucket: 'validate-logs',
  tracesBucket: 'validate-traces',
});
validateRegionalArn(alarmTopicArn, 'sns', region);
const ciIngestSecretArn =
  stack === 'prod' ? config.require('ciIngestSecretArn') : undefined;
if (ciIngestSecretArn)
  validateRegionalArn(ciIngestSecretArn, 'secretsmanager', 'us-east-1');
export const ciTelemetryRoleArn = ciIngestSecretArn
  ? createCiTelemetryRole(ciIngestSecretArn)
  : undefined;
const vpc = createNetwork(region, tags);
const subnet = vpc.privateSubnet;
const subnetId = subnet.id;

// Route53 is global; issuance and use of the certificate are both in Ohio.
const zone = aws.route53.getZoneOutput({
  name: baseDomain,
  privateZone: false,
});
const certificate = new aws.acm.Certificate('observability', {
  domainName: grafanaHost,
  subjectAlternativeNames: [otlpHost, oncallHost],
  validationMethod: 'DNS',
  tags,
});
const validationRecords = [grafanaHost, otlpHost, oncallHost].map(
  (hostname) => {
    const option = certificate.domainValidationOptions.apply((options) => {
      const match = options.find((entry) => entry.domainName === hostname);
      if (!match) throw new Error(`Missing ACM validation for ${hostname}`);
      return match;
    });
    return new aws.route53.Record(`observability-certificate-${hostname}`, {
      zoneId: zone.zoneId,
      name: option.resourceRecordName,
      type: option.resourceRecordType,
      records: [option.resourceRecordValue],
      ttl: 300,
    });
  }
);
const validatedCertificate = new aws.acm.CertificateValidation(
  'observability',
  {
    certificateArn: certificate.arn,
    validationRecordFqdns: validationRecords.map((record) => record.fqdn),
  }
);

function telemetryBucket(name: string) {
  const bucket = new aws.s3.BucketV2(
    name,
    { forceDestroy: false, tags },
    durable
  );
  new aws.s3.BucketPublicAccessBlock(`${name}-private`, {
    bucket: bucket.id,
    blockPublicAcls: true,
    blockPublicPolicy: true,
    ignorePublicAcls: true,
    restrictPublicBuckets: true,
  });
  new aws.s3.BucketServerSideEncryptionConfigurationV2(`${name}-encryption`, {
    bucket: bucket.id,
    rules: [{ applyServerSideEncryptionByDefault: { sseAlgorithm: 'AES256' } }],
  });
  new aws.s3.BucketPolicy(`${name}-tls`, {
    bucket: bucket.id,
    policy: bucket.arn.apply((arn) =>
      JSON.stringify({
        Version: '2012-10-17',
        Statement: [
          {
            Effect: 'Deny',
            Principal: '*',
            Action: 's3:*',
            Resource: [arn, `${arn}/*`],
            Condition: { Bool: { 'aws:SecureTransport': 'false' } },
          },
        ],
      })
    ),
  });
  // Loki/Tempo own retention. Do not expire active blocks with S3 lifecycle.
  new aws.s3.BucketLifecycleConfigurationV2(`${name}-multipart`, {
    bucket: bucket.id,
    rules: [
      {
        id: 'abort-incomplete-uploads',
        status: 'Enabled',
        filter: { prefix: '' },
        abortIncompleteMultipartUpload: { daysAfterInitiation: 1 },
      },
    ],
  });
  return bucket;
}
const logs = telemetryBucket('observability-logs');
const traces = telemetryBucket('observability-traces');
const data = new aws.ebs.Volume(
  'observability-data',
  {
    availabilityZone: subnet.availabilityZone,
    type: 'gp3',
    size: 300,
    encrypted: true,
    snapshotId: config.get('dataSnapshotId'),
    tags: { ...tags, ObservabilityBackup: stack },
  },
  durable
);

const role = new aws.iam.Role('observability-host', {
  assumeRolePolicy: aws.iam.assumeRolePolicyForPrincipal({
    Service: 'ec2.amazonaws.com',
  }),
  tags,
});
const ssm = new aws.iam.RolePolicyAttachment('observability-ssm', {
  role: role.name,
  policyArn: 'arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore',
});
const policy = new aws.iam.RolePolicy('observability-storage', {
  role: role.id,
  policy: pulumi.all([logs.arn, traces.arn]).apply((arns) =>
    JSON.stringify({
      Version: '2012-10-17',
      Statement: [
        {
          Effect: 'Allow',
          Action: [
            's3:ListBucket',
            's3:GetBucketLocation',
            's3:ListBucketMultipartUploads',
          ],
          Resource: arns,
        },
        {
          Effect: 'Allow',
          Action: [
            's3:GetObject',
            's3:PutObject',
            's3:DeleteObject',
            's3:AbortMultipartUpload',
            's3:ListMultipartUploadParts',
          ],
          Resource: arns.map((arn) => `${arn}/*`),
        },
        {
          Effect: 'Allow',
          Action: ['secretsmanager:GetSecretValue'],
          Resource: secretArn,
        },
        {
          Effect: 'Allow',
          Action: [
            'cloudwatch:ListMetrics',
            'cloudwatch:GetMetricData',
            'cloudwatch:GetMetricStatistics',
          ],
          Resource: '*',
          Condition: {
            StringEquals: { 'aws:RequestedRegion': ['us-east-1', 'us-east-2'] },
          },
        },
        {
          Effect: 'Allow',
          Action: ['sns:Publish'],
          Resource: alarmTopicArn,
        },
        {
          Effect: 'Allow',
          Action: ['cloudwatch:PutMetricData'],
          Resource: '*',
          Condition: {
            StringEquals: { 'cloudwatch:namespace': 'Macro/Observability' },
          },
        },
      ],
    })
  ),
});
const profile = new aws.iam.InstanceProfile('observability-host', {
  role: role.name,
});
const albSg = new aws.ec2.SecurityGroup('observability-alb', {
  vpcId: vpc.vpcId,
  description: 'Public HTTPS only',
  ingress: [
    { protocol: 'tcp', fromPort: 443, toPort: 443, cidrBlocks: ['0.0.0.0/0'] },
  ],
  tags,
});
const hostSg = new aws.ec2.SecurityGroup('observability-host', {
  vpcId: vpc.vpcId,
  description: 'Only the ALB can reach the proxy; administration via SSM',
  ingress: [
    {
      protocol: 'tcp',
      fromPort: 8080,
      toPort: 8080,
      securityGroups: [albSg.id],
    },
  ],
  egress: [
    { protocol: '-1', fromPort: 0, toPort: 0, cidrBlocks: ['0.0.0.0/0'] },
  ],
  tags,
});
new aws.ec2.SecurityGroupRule('observability-alb-to-host', {
  securityGroupId: albSg.id,
  type: 'egress',
  protocol: 'tcp',
  fromPort: 8080,
  toPort: 8080,
  sourceSecurityGroupId: hostSg.id,
});
const instance = new aws.ec2.Instance(
  'observability',
  {
    ami: amiId,
    instanceType: 'm7i.xlarge',
    subnetId,
    associatePublicIpAddress: false,
    vpcSecurityGroupIds: [hostSg.id],
    iamInstanceProfile: profile.name,
    metadataOptions: {
      httpEndpoint: 'enabled',
      httpTokens: 'required',
      httpPutResponseHopLimit: 1,
    },
    rootBlockDevice: {
      volumeType: 'gp3',
      volumeSize: 30,
      encrypted: true,
      deleteOnTermination: true,
    },
    userDataReplaceOnChange: true,
    userDataBase64: pulumi
      .all([data.id, logs.bucket, traces.bucket])
      .apply(([volumeId, logsBucket, tracesBucket]) =>
        renderUserData({
          region,
          grafanaHost,
          otlpHost,
          oncallHost,
          secretArn,
          alarmTopicArn,
          volumeId,
          logsBucket,
          tracesBucket,
        })
      ),
    tags,
  },
  { deleteBeforeReplace: true, dependsOn: [policy, ssm, ...vpc.privateReady] }
);
new aws.ec2.VolumeAttachment('observability-data', {
  deviceName: '/dev/sdf',
  volumeId: data.id,
  instanceId: instance.id,
  stopInstanceBeforeDetaching: true,
  forceDetach: false,
});

const alb = new aws.lb.LoadBalancer(
  'observability',
  {
    loadBalancerType: 'application',
    internal: false,
    subnets: vpc.publicSubnetIds,
    securityGroups: [albSg.id],
    dropInvalidHeaderFields: true,
    desyncMitigationMode: 'strictest',
    tags,
  },
  { dependsOn: vpc.publicReady }
);
const target = new aws.lb.TargetGroup('observability', {
  port: 8080,
  protocol: 'HTTP',
  targetType: 'instance',
  vpcId: vpc.vpcId,
  healthCheck: { path: '/healthz', matcher: '200', interval: 30 },
  deregistrationDelay: 30,
  tags,
});
new aws.lb.TargetGroupAttachment('observability', {
  targetGroupArn: target.arn,
  targetId: instance.id,
  port: 8080,
});
const listener = new aws.lb.Listener('observability-https', {
  loadBalancerArn: alb.arn,
  port: 443,
  protocol: 'HTTPS',
  certificateArn: validatedCertificate.certificateArn,
  sslPolicy: 'ELBSecurityPolicy-TLS13-1-2-2021-06',
  defaultActions: [
    {
      type: 'fixed-response',
      fixedResponse: { contentType: 'text/plain', statusCode: '404' },
    },
  ],
});
new aws.lb.ListenerRule('observability-hosts', {
  listenerArn: listener.arn,
  priority: 1,
  conditions: [{ hostHeader: { values: [grafanaHost, otlpHost] } }],
  actions: [{ type: 'forward', targetGroupArn: target.arn }],
});
// Same nginx port, but its own health check so a GoAlert outage alarms
// separately from Grafana.
const oncallTarget = new aws.lb.TargetGroup('observability-oncall', {
  port: 8080,
  protocol: 'HTTP',
  targetType: 'instance',
  vpcId: vpc.vpcId,
  healthCheck: { path: '/goalert-healthz', matcher: '200', interval: 30 },
  deregistrationDelay: 30,
  tags,
});
new aws.lb.TargetGroupAttachment('observability-oncall', {
  targetGroupArn: oncallTarget.arn,
  targetId: instance.id,
  port: 8080,
});
new aws.lb.ListenerRule('observability-oncall', {
  listenerArn: listener.arn,
  priority: 2,
  conditions: [{ hostHeader: { values: [oncallHost] } }],
  actions: [{ type: 'forward', targetGroupArn: oncallTarget.arn }],
});
for (const hostname of [grafanaHost, otlpHost, oncallHost]) {
  new aws.route53.Record(hostname, {
    zoneId: zone.zoneId,
    name: hostname,
    type: 'A',
    aliases: [
      { name: alb.dnsName, zoneId: alb.zoneId, evaluateTargetHealth: true },
    ],
  });
}

const backupRole = new aws.iam.Role('observability-snapshots', {
  assumeRolePolicy: aws.iam.assumeRolePolicyForPrincipal({
    Service: 'dlm.amazonaws.com',
  }),
  tags,
});
const backupPolicy = new aws.iam.RolePolicyAttachment(
  'observability-snapshots',
  {
    role: backupRole.name,
    policyArn:
      'arn:aws:iam::aws:policy/service-role/AWSDataLifecycleManagerServiceRole',
  }
);
new aws.dlm.LifecyclePolicy(
  'observability-snapshots',
  {
    description: 'Daily crash-consistent data volume snapshots retained 7 days',
    executionRoleArn: backupRole.arn,
    state: 'ENABLED',
    policyDetails: {
      resourceTypes: ['VOLUME'],
      targetTags: { ObservabilityBackup: stack },
      schedules: [
        {
          name: 'daily',
          copyTags: true,
          createRule: { interval: 24, intervalUnit: 'HOURS', times: '04:00' },
          retainRule: { count: 7 },
        },
      ],
    },
    tags,
  },
  { dependsOn: [backupPolicy] }
);
const alarm = {
  comparisonOperator: 'GreaterThanThreshold',
  evaluationPeriods: 2,
  period: 60,
  statistic: 'Maximum',
  alarmActions: [alarmTopicArn],
  okActions: [alarmTopicArn],
  tags,
};
new aws.cloudwatch.MetricAlarm('observability-instance', {
  ...alarm,
  namespace: 'AWS/EC2',
  metricName: 'StatusCheckFailed',
  threshold: 0,
  dimensions: { InstanceId: instance.id },
  treatMissingData: 'breaching',
});
new aws.cloudwatch.MetricAlarm('observability-ui', {
  ...alarm,
  namespace: 'AWS/ApplicationELB',
  metricName: 'UnHealthyHostCount',
  threshold: 0,
  dimensions: { LoadBalancer: alb.arnSuffix, TargetGroup: target.arnSuffix },
  treatMissingData: 'breaching',
});
// Paging must not depend on GoAlert to report that GoAlert is down. On-call
// phones subscribe to this topic by SMS directly (see README).
new aws.cloudwatch.MetricAlarm('observability-goalert', {
  ...alarm,
  alarmDescription: `GoAlert at ${oncallHost} is failing health checks; pages are not being sent.`,
  namespace: 'AWS/ApplicationELB',
  metricName: 'UnHealthyHostCount',
  threshold: 0,
  dimensions: {
    LoadBalancer: alb.arnSuffix,
    TargetGroup: oncallTarget.arnSuffix,
  },
  treatMissingData: 'breaching',
});
for (const [name, path] of [
  ['observability-disk', '/srv/observability'],
  ['observability-root-disk', '/'],
]) {
  new aws.cloudwatch.MetricAlarm(name, {
    ...alarm,
    namespace: 'Macro/Observability',
    metricName: 'disk_used_percent',
    threshold: 80,
    dimensions: { InstanceId: instance.id, path, fstype: 'ext4' },
    period: 300,
    treatMissingData: 'breaching',
  });
}
new aws.cloudwatch.MetricAlarm('observability-memory', {
  ...alarm,
  namespace: 'Macro/Observability',
  metricName: 'mem_used_percent',
  threshold: 90,
  dimensions: { InstanceId: instance.id },
  period: 300,
  treatMissingData: 'breaching',
});

export const grafanaUrl = `https://${grafanaHost}`;
export const otlpHttpEndpoint = `https://${otlpHost}`;
export const goalertUrl = `https://${oncallHost}`;
export const instanceId = instance.id;
export const dataVolumeId = data.id;
export const logsBucket = logs.bucket;
export const tracesBucket = traces.bucket;
export const observabilityRegion = region;