Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

chore(infra): dev PostgreSQL monitoring with a native NixOS collector (#7989)

Wolf Mermelsteinbda0fc186fa5
diff --git a/infra/bun.lock b/infra/bun.lock
index 550909f839..c8dac7540c 100644
--- a/infra/bun.lock
+++ b/infra/bun.lock
@@ -92,6 +92,10 @@
"name": "convert-service-stack",
"version": "0.0.0",
},
+ "stacks/database-observability": {
+ "name": "database-observability-stack",
+ "version": "0.0.0",
+ },
"stacks/deleted-item-poller": {
"name": "deleted-item-poller-stack",
"version": "0.0.0",
@@ -569,6 +573,8 @@
"cssesc": ["[email protected]", "", { "bin": { "cssesc": "bin/cssesc" } }, "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg=="],
+ "database-observability-stack": ["database-observability-stack@workspace:stacks/database-observability"],
+
"debug": ["[email protected]", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
"deleted-item-poller-stack": ["deleted-item-poller-stack@workspace:stacks/deleted-item-poller"],
diff --git a/infra/stacks/database-observability/Pulumi.yaml b/infra/stacks/database-observability/Pulumi.yaml
new file mode 100644
index 0000000000..1ef49b7cee
--- /dev/null
+++ b/infra/stacks/database-observability/Pulumi.yaml
@@ -0,0 +1,3 @@
+name: database-observability
+runtime: nodejs
+description: Native NixOS PostgreSQL metrics collector in the database region
diff --git a/infra/stacks/database-observability/README.md b/infra/stacks/database-observability/README.md
new file mode 100644
index 0000000000..81dd0ac488
--- /dev/null
+++ b/infra/stacks/database-observability/README.md
@@ -0,0 +1,97 @@
+# PostgreSQL metrics pilot
+
+This stack adds one native NixOS/Alloy collector in a private Virginia subnet,
+with no inbound ports or public IP. It monitors the dev writer and replica over
+verified PostgreSQL TLS and forwards metrics to the existing Ohio Prometheus.
+Datadog, application tasks, RDS parameter groups, and production are unchanged.
+
+This is the database health baseline. It does not collect SQL text, query
+fingerprints, samples or execution plans, change existing alerts, or provide
+Grafana Cloud's Database Observability UI. PostgreSQL 14 and 18 compatibility
+must be verified before expanding the dev-only stack to production.
+
+## Deployment order
+
+1. Build and publish a new **Ohio observability** AMI from
+ `infra/stacks/observability`; deploy it first. This adds
+ `https://otlp.macro-internal.com/api/v1/write`, an exact POST-only endpoint
+ protected by HTTP Basic (`metrics`, existing OTLP token) over ALB TLS. It
+ exposes no query/administration routes. Existing OTLP bearer access is
+ unchanged. The password is hashed at boot and passed privately to nginx.
+ Grafana gets the provisioned **Macro PostgreSQL** dashboard at
+ `/d/macro-postgres`.
+2. As an authorized database administrator, connect **to dev writer / macrodb**
+ with TLS and run `psql --file monitoring-role.sql`. Choose a random password
+ of at least 32 characters at its secure prompt. The role/grants replicate to
+ the reader. The script grants statistics access via `pg_monitor`, not
+ application-table SELECT; it sets connection and query timeouts and defaults
+ to read-only transactions. It doesn't install extensions or restart RDS.
+ `pg_monitor` can read activity/query information, so treat this credential as
+ sensitive even though this collector does not export query text.
+3. Save that password in Doppler `observability/dev` as
+ `GRAFANA_POSTGRES_PASSWORD` and an AWS-managed-key Secrets Manager secret in
+ Virginia with JSON shape `{"password":"..."}`. Use the existing raw-string
+ dev-ingest secret for `ingestSecretArn`. Doppler is the operator source;
+ the host retrieves only the two explicit AWS secrets using its instance
+ role. There is no automatic Doppler-to-Secrets-Manager synchronization.
+ Never put values in Pulumi config, command-line arguments or user data.
+4. Build the pinned collector image:
+ `nix build ./infra/stacks/observability#database-collector --out-link result-database-collector`.
+ Publish with
+ `python3 infra/stacks/observability/nixos/publish-image.py result-database-collector VIRGINIA_PRIVATE_IMAGE_BUCKET --region us-east-1 --name-prefix macro-database-collector`.
+ The image is fixed; EC2 user data carries validated nonsecret settings only.
+5. Initialize the separate Pulumi project here with a `dev` stack. Configure
+ `aws:region=us-east-1`, `amiId`, `subnetId`, `secretArn`, and `ingestSecretArn`.
+ The subnet must be private, in the database VPC, with outbound NAT/HTTPS
+ access to AWS APIs and Ohio. The AMI must be the reviewed collector image
+ owned by our account. The stack discovers dev writer/replica endpoints and
+ adds port 5432 rules allowing traffic from the collector security group to
+ their existing security groups. These rules cover every resource attached
+ to those destination groups; verify their membership during preview.
+ Run `pulumi preview --stack dev`, review only additive collector resources,
+ then deploy. No database resource belongs to this stack.
+
+The stack deliberately requires a reviewed image and secret references, rather
+than a fake AMI, generated admin password or automatic database mutation.
+Use AWS-managed encryption for the two secrets; a customer-managed key would
+also need narrowly scoped `kms:Decrypt` permission.
+
+## Verification before relying on it
+
+Check SSM connectivity and `database-collector-bootstrap` / `alloy` units. Each
+endpoint must report `pg_up=1`, `pg_exporter_last_scrape_error=0`, and scrape
+`up=1` with distinct `db_instance` and `db_role` labels. Inspect the dashboard's
+transaction, connection, lock and vacuum metrics during representative dev
+traffic. Missing metrics are not replaced with reassuring zeros. The freshness
+panel measures the latest sample's age within Prometheus's five-minute lookback;
+after that it shows no data, not an increasing outage timer. Verify no passwords,
+connection-string labels, SQL statements or parameter values appear
+in ingested data or logs. The exporter runs once per minute with five-second
+SQL timeouts; a failing collector does not imply the database is down.
+
+Exercise a temporary receiver outage in an isolated smoke environment, restore
+it and verify remote-write catch-up. Alloy self-metrics expose pending/failed
+samples. No alert notifications are enabled in this PR; collection-health
+alert routing and independent host/disk alarms are follow-up rollout work.
+This pilot must not be the sole database monitor.
+
+## Storage, rotation and rollback
+
+The encrypted 20 GiB gp3 data volume is retained/protected across host
+replacement; stable ownership and volume-ID verification prevent silent
+fallback to root storage. Prometheus remote-write WAL is retained up to 24 hours,
+not an unlimited queue or a guarantee against a whole Virginia failure. Monitor
+volume capacity, measure actual telemetry volume and tune the retention before
+production. Metrics delivered to Ohio use its existing Prometheus retention.
+
+For credential rotation, update the role password and AWS secret, then restart
+`database-collector-bootstrap.service` over SSM. Alloy stops with bootstrap and
+starts with new private systemd credentials; the WAL remains. A partial failure
+leaves collection stopped and retrying. Rotating the shared ingest token also
+requires updating its Ohio source and all existing OTLP clients.
+
+Rollback by stopping Alloy or removing this collector stack and its added
+security-group rules; the WAL volume remains protected. The role may be removed
+by an administrator after its last consumer is gone. Keep Datadog running during
+all pilot work. Query statistics, richer investigations, full Datadog inventory
+and alert-by-alert migration remain later passes.
diff --git a/infra/stacks/database-observability/index.ts b/infra/stacks/database-observability/index.ts
new file mode 100644
index 0000000000..111b25feb5
--- /dev/null
+++ b/infra/stacks/database-observability/index.ts
@@ -0,0 +1,170 @@
+import { gzipSync } from 'node:zlib';
+import * as aws from '@pulumi/aws';
+import * as pulumi from '@pulumi/pulumi';
+
+const config = new pulumi.Config();
+const stack = pulumi.getStack();
+const region = aws.config.requireRegion();
+if (stack !== 'dev' || region !== 'us-east-1') {
+ throw new Error('The database collector pilot is dev-only in us-east-1');
+}
+const amiId = config.require('amiId');
+const subnetId = config.require('subnetId');
+const secretArn = config.require('secretArn');
+const ingestSecretArn = config.require('ingestSecretArn');
+for (const arn of [secretArn, ingestSecretArn]) {
+ if (
+ !/^arn:aws:secretsmanager:us-east-1:569036502058:secret:[a-zA-Z0-9/_+=.@-]+$/.test(
+ arn
+ )
+ ) {
+ throw new Error('Secrets must belong to Macro in us-east-1');
+ }
+}
+if (!/^ami-[a-f0-9]+$/.test(amiId) || !/^subnet-[a-f0-9]+$/.test(subnetId)) {
+ throw new Error(
+ 'Configure a reviewed collector AMI and a private database-VPC subnet'
+ );
+}
+const tags = { project: 'database-observability', environment: stack };
+const subnet = aws.ec2.getSubnetOutput({ id: subnetId });
+const databases = ['macro-db-dev', 'macro-db-dev-read-replica'].map((name) =>
+ aws.rds.getInstanceOutput({ dbInstanceIdentifier: name })
+);
+const group = new aws.ec2.SecurityGroup('database-collector', {
+ vpcId: subnet.vpcId,
+ description:
+ 'No inbound access; SSM administration and outbound database metrics',
+ tags,
+});
+const https = new aws.vpc.SecurityGroupEgressRule('database-collector-https', {
+ securityGroupId: group.id,
+ cidrIpv4: '0.0.0.0/0',
+ ipProtocol: 'tcp',
+ fromPort: 443,
+ toPort: 443,
+ tags,
+});
+const databaseRules = pulumi
+ .all(databases.map((db) => db.vpcSecurityGroups))
+ .apply((groups) =>
+ [...new Set(groups.flat())].flatMap((securityGroupId) => [
+ new aws.vpc.SecurityGroupEgressRule(
+ `collector-postgres-out-${securityGroupId}`,
+ {
+ securityGroupId: group.id,
+ referencedSecurityGroupId: securityGroupId,
+ ipProtocol: 'tcp',
+ fromPort: 5432,
+ toPort: 5432,
+ tags,
+ }
+ ),
+ new aws.vpc.SecurityGroupIngressRule(
+ `database-collector-in-${securityGroupId}`,
+ {
+ securityGroupId,
+ referencedSecurityGroupId: group.id,
+ ipProtocol: 'tcp',
+ fromPort: 5432,
+ toPort: 5432,
+ tags,
+ }
+ ),
+ ])
+ );
+const volume = new aws.ebs.Volume(
+ 'database-collector-data',
+ {
+ availabilityZone: subnet.availabilityZone,
+ type: 'gp3',
+ size: 20,
+ encrypted: true,
+ tags,
+ },
+ { protect: true, retainOnDelete: true }
+);
+const role = new aws.iam.Role('database-collector', {
+ assumeRolePolicy: aws.iam.assumeRolePolicyForPrincipal({
+ Service: 'ec2.amazonaws.com',
+ }),
+ tags,
+});
+const ssm = new aws.iam.RolePolicyAttachment('database-collector-ssm', {
+ role: role.name,
+ policyArn: 'arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore',
+});
+const secrets = new aws.iam.RolePolicy('database-collector-secrets', {
+ role: role.name,
+ policy: JSON.stringify({
+ Version: '2012-10-17',
+ Statement: [
+ {
+ Effect: 'Allow',
+ Action: 'secretsmanager:GetSecretValue',
+ Resource: [secretArn, ingestSecretArn],
+ },
+ ],
+ }),
+});
+const profile = new aws.iam.InstanceProfile('database-collector', {
+ role: role.name,
+});
+const instance = new aws.ec2.Instance(
+ 'database-collector',
+ {
+ ami: amiId,
+ instanceType: 't3.small',
+ subnetId,
+ associatePublicIpAddress: false,
+ vpcSecurityGroupIds: [group.id],
+ iamInstanceProfile: profile.name,
+ metadataOptions: {
+ httpEndpoint: 'enabled',
+ httpTokens: 'required',
+ httpPutResponseHopLimit: 1,
+ },
+ rootBlockDevice: {
+ volumeType: 'gp3',
+ volumeSize: 12,
+ encrypted: true,
+ deleteOnTermination: true,
+ },
+ userDataReplaceOnChange: true,
+ userDataBase64: pulumi
+ .all([volume.id, ...databases.map((db) => db.address)])
+ .apply(([volumeId, writerHost, replicaHost]) =>
+ gzipSync(
+ JSON.stringify({
+ version: 1,
+ region,
+ environment: stack,
+ volumeId,
+ secretArn,
+ ingestSecretArn,
+ writerHost,
+ replicaHost,
+ })
+ ).toString('base64')
+ ),
+ tags,
+ },
+ {
+ deleteBeforeReplace: true,
+ dependsOn: databaseRules.apply((rules) => [ssm, secrets, https, ...rules]),
+ }
+);
+new aws.ec2.VolumeAttachment(
+ 'database-collector-data',
+ {
+ deviceName: '/dev/sdf',
+ volumeId: volume.id,
+ instanceId: instance.id,
+ stopInstanceBeforeDetaching: true,
+ forceDetach: false,
+ },
+ { deleteBeforeReplace: true }
+);
+export const instanceId = instance.id;
+export const dataVolumeId = volume.id;
+export const securityGroupId = group.id;
diff --git a/infra/stacks/database-observability/monitoring-role.sql b/infra/stacks/database-observability/monitoring-role.sql
new file mode 100644
index 0000000000..84edce5fa2
--- /dev/null
+++ b/infra/stacks/database-observability/monitoring-role.sql
@@ -0,0 +1,19 @@
+-- Run with psql as an authorized database administrator on the DEV writer,
+-- connected to macrodb. Roles and grants replicate; do not run on the replica.
+\set ON_ERROR_STOP on
+DO $$
+BEGIN
+ IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'grafana_monitor') THEN
+ CREATE ROLE grafana_monitor LOGIN;
+ END IF;
+END $$;
+ALTER ROLE grafana_monitor NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION
+ NOBYPASSRLS CONNECTION LIMIT 5;
+GRANT CONNECT ON DATABASE macrodb TO grafana_monitor;
+GRANT pg_monitor TO grafana_monitor;
+ALTER ROLE grafana_monitor SET default_transaction_read_only = on;
+ALTER ROLE grafana_monitor SET statement_timeout = '5s';
+ALTER ROLE grafana_monitor SET lock_timeout = '1s';
+ALTER ROLE grafana_monitor SET idle_in_transaction_session_timeout = '10s';
+-- psql prompts securely; no password is embedded in this file or command history.
+\password grafana_monitor
diff --git a/infra/stacks/database-observability/package.json b/infra/stacks/database-observability/package.json
new file mode 100644
index 0000000000..515bc55354
--- /dev/null
+++ b/infra/stacks/database-observability/package.json
@@ -0,0 +1 @@
+{"name":"database-observability-stack","version":"0.0.0","private":true,"main":"index.ts"}
diff --git a/infra/stacks/database-observability/tsconfig.json b/infra/stacks/database-observability/tsconfig.json
new file mode 100644
index 0000000000..76929fa150
--- /dev/null
+++ b/infra/stacks/database-observability/tsconfig.json
@@ -0,0 +1 @@
+{"extends":"../../tsconfig.json","include":["index.ts"]}
diff --git a/infra/stacks/observability/assets/refresh-secrets.py b/infra/stacks/observability/assets/refresh-secrets.py
index 404640c16b..dde22d99f0 100644
--- a/infra/stacks/observability/assets/refresh-secrets.py
+++ b/infra/stacks/observability/assets/refresh-secrets.py
@@ -23,6 +23,13 @@ for key in keys:
for key in ['grafana_secret_key', 'otlp_token']:
if len(values[key]) < 32:
raise SystemExit(f'Secret too short: {key}')
+# Native remote-write uses HTTP Basic over TLS. Only the password hash reaches nginx.
+hashed = subprocess.run(
+ ['openssl', 'passwd', '-6', '-stdin'], input=values['otlp_token'] + '\n',
+ check=True, capture_output=True, text=True,
+).stdout.strip()
+values['metrics.htpasswd'] = 'metrics:' + hashed + '\n'
+keys.append('metrics.htpasswd')
directory = Path('/run/macro-observability')
directory.mkdir(mode=0o700, exist_ok=True)
os.chown(directory, 0, 0)
diff --git a/infra/stacks/observability/flake.nix b/infra/stacks/observability/flake.nix
index 073cfffa19..d3285f921e 100644
--- a/infra/stacks/observability/flake.nix
+++ b/infra/stacks/observability/flake.nix
@@ -13,12 +13,20 @@
./nixos/host.nix
];
};
+ databaseCollector = nixpkgs.lib.nixosSystem {
+ inherit system;
+ modules = [
+ "${nixpkgs}/nixos/maintainers/scripts/ec2/amazon-image.nix"
+ ./nixos/database-collector/host.nix
+ ];
+ };
in
{
nixosConfigurations.observability = host;
+ nixosConfigurations.database-collector = databaseCollector;
packages.${system} = {
default = host.config.system.build.amazonImage;
-
+ database-collector = databaseCollector.config.system.build.amazonImage;
};
formatter.${system} = nixpkgs.legacyPackages.${system}.nixfmt;
};
diff --git a/infra/stacks/observability/nixos/database-collector/bootstrap.py b/infra/stacks/observability/nixos/database-collector/bootstrap.py
new file mode 100644
index 0000000000..04ce4f1fc0
--- /dev/null
+++ b/infra/stacks/observability/nixos/database-collector/bootstrap.py
@@ -0,0 +1,88 @@
+"""Read nonsecret EC2 metadata and retrieve the collector's runtime credentials."""
+import gzip
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import urllib.parse
+import urllib.request
+
+
+def metadata():
+ http = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+ base = 'http://169.254.169.254/latest/'
+ request = urllib.request.Request(base + 'api/token', method='PUT',
+ headers={'X-aws-ec2-metadata-token-ttl-seconds': '60'})
+ with http.open(request, timeout=10) as response:
+ token = response.read().decode()
+ request = urllib.request.Request(base + 'user-data',
+ headers={'X-aws-ec2-metadata-token': token})
+ with http.open(request, timeout=10) as response:
+ raw = response.read(16385)
+ if len(raw) > 16384:
+ raise ValueError('User data exceeds EC2 limit')
+ return validate(json.loads(gzip.decompress(raw)))
+
+
+def validate(settings):
+ patterns = {
+ 'region': r'us-east-1', 'environment': r'dev',
+ 'volumeId': r'vol-[a-f0-9]+',
+ 'secretArn': r'arn:aws:secretsmanager:us-east-1:569036502058:secret:[a-zA-Z0-9/_+=.@-]+',
+ 'ingestSecretArn': r'arn:aws:secretsmanager:us-east-1:569036502058:secret:[a-zA-Z0-9/_+=.@-]+',
+ 'writerHost': r'macro-db-dev\.[a-z0-9]+\.us-east-1\.rds\.amazonaws\.com',
+ 'replicaHost': r'macro-db-dev-read-replica\.[a-z0-9]+\.us-east-1\.rds\.amazonaws\.com',
+ }
+ if not isinstance(settings, dict) or set(settings) != set(patterns) | {'version'} or settings['version'] != 1:
+ raise ValueError('Unsupported database collector metadata')
+ for key, pattern in patterns.items():
+ if not isinstance(settings[key], str) or not re.fullmatch(pattern, settings[key]):
+ raise ValueError(f'Invalid collector setting: {key}')
+ return settings
+
+
+def secret(arn):
+ result = subprocess.run(['aws', 'secretsmanager', 'get-secret-value',
+ '--region', 'us-east-1', '--secret-id', arn, '--output', 'json'],
+ check=True, capture_output=True, text=True)
+ return json.loads(result.stdout)['SecretString']
+
+
+def main():
+ settings = metadata()
+ root = Path('/opt/observability')
+ root.mkdir(mode=0o755, parents=True, exist_ok=True)
+ (root / 'volume-id').write_text(settings['volumeId'])
+ password = json.loads(secret(settings['secretArn']))['password']
+ token = secret(settings['ingestSecretArn'])
+ for name, value in [('password', password), ('ingest token', token)]:
+ if not isinstance(value, str) or len(value) < 32 or value.strip() != value or '\n' in value or '\r' in value or '\x00' in value:
+ raise ValueError(f'Invalid {name}')
+ options = urllib.parse.urlencode({
+ 'sslmode': 'verify-full', 'sslrootcert': '/etc/ssl/certs/rds-us-east-1.pem',
+ 'connect_timeout': '5', 'application_name': 'grafana_database_collector',
+ 'options': '-c statement_timeout=5000 -c lock_timeout=1000 -c default_transaction_read_only=on',
+ })
+ values = {'otlp_token': token}
+ for target in ['writer', 'replica']:
+ values[target + '_dsn'] = (
+ 'postgresql://grafana_monitor:' + urllib.parse.quote(password, safe='')
+ + '@' + settings[target + 'Host'] + ':5432/macrodb?' + options
+ )
+ directory = Path('/run/macro-database-collector')
+ directory.mkdir(mode=0o700, exist_ok=True)
+ for name, value in values.items():
+ path = directory / name
+ path.unlink(missing_ok=True)
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o400)
+ with os.fdopen(fd, 'w') as output:
+ output.write(value)
+
+
+if __name__ == '__main__':
+ try:
+ main()
+ except Exception:
+ # SDK/database errors may include credentials; never log exception payloads.
+ raise SystemExit('Database collector bootstrap failed; check metadata, secret schema and IAM access') from None
diff --git a/infra/stacks/observability/nixos/database-collector/collector.alloy b/infra/stacks/observability/nixos/database-collector/collector.alloy
new file mode 100644
index 0000000000..2aec89729a
--- /dev/null
+++ b/infra/stacks/observability/nixos/database-collector/collector.alloy
@@ -0,0 +1,113 @@
+local.file "writer_dsn" {
+ filename = sys.env("CREDENTIALS_DIRECTORY") + "/writer_dsn"
+ is_secret = true
+}
+
+local.file "replica_dsn" {
+ filename = sys.env("CREDENTIALS_DIRECTORY") + "/replica_dsn"
+ is_secret = true
+}
+
+// A separate exporter per endpoint avoids upstream multi-DSN collector limitations.
+// Statement samples, SQL text and execution plans are deliberately a later pass.
+prometheus.exporter.postgres "writer" {
+ data_source_names = [local.file.writer_dsn.content]
+ enabled_collectors = ["database", "locks", "replication", "replication_slot", "stat_bgwriter", "stat_database", "stat_progress_vacuum", "stat_user_tables", "statio_user_tables", "wal", "long_running_transactions", "stat_activity_autovacuum"]
+}
+
+prometheus.exporter.postgres "replica" {
+ data_source_names = [local.file.replica_dsn.content]
+ enabled_collectors = ["database", "locks", "replication", "replication_slot", "stat_bgwriter", "stat_database", "stat_progress_vacuum", "stat_user_tables", "statio_user_tables", "wal", "long_running_transactions", "stat_activity_autovacuum", "stat_wal_receiver"]
+}
+
+discovery.relabel "writer" {
+ targets = prometheus.exporter.postgres.writer.targets
+
+ rule {
+ target_label = "db_instance"
+ replacement = "macro-db-dev"
+ }
+
+ rule {
+ target_label = "db_role"
+ replacement = "writer"
+ }
+}
+
+discovery.relabel "replica" {
+ targets = prometheus.exporter.postgres.replica.targets
+
+ rule {
+ target_label = "db_instance"
+ replacement = "macro-db-dev-read-replica"
+ }
+
+ rule {
+ target_label = "db_role"
+ replacement = "replica"
+ }
+}
+
+prometheus.scrape "postgres" {
+ targets = array.concat(discovery.relabel.writer.output, discovery.relabel.replica.output)
+ job_name = "integrations/postgres"
+ scrape_interval = "60s"
+ scrape_timeout = "15s"
+ forward_to = [prometheus.relabel.database.receiver]
+}
+
+prometheus.relabel "database" {
+ // Don't retain exporter connection strings as labels, even if redacted upstream.
+ rule {
+ action = "labeldrop"
+ regex = "server"
+ }
+
+ rule {
+ source_labels = ["db_instance"]
+ target_label = "instance"
+ }
+ forward_to = [prometheus.remote_write.ohio.receiver]
+}
+
+prometheus.exporter.self "collector" { }
+
+discovery.relabel "collector" {
+ targets = prometheus.exporter.self.collector.targets
+
+ rule {
+ target_label = "job"
+ replacement = "observability/database-collector"
+ }
+}
+
+prometheus.scrape "collector" {
+ targets = discovery.relabel.collector.output
+ job_name = "observability/database-collector"
+ scrape_interval = "30s"
+ forward_to = [prometheus.remote_write.ohio.receiver]
+}
+
+prometheus.remote_write "ohio" {
+ external_labels = {deployment_environment = "dev", aws_region = "us-east-1"}
+
+ endpoint {
+ url = "https://otlp.macro-internal.com/api/v1/write"
+
+ basic_auth {
+ username = "metrics"
+ password_file = sys.env("CREDENTIALS_DIRECTORY") + "/otlp_token"
+ }
+
+ queue_config {
+ max_shards = 2
+ capacity = 2500
+ max_samples_per_send = 1000
+ }
+ }
+
+ wal {
+ min_keepalive_time = "5m"
+ max_keepalive_time = "24h"
+ }
+}
diff --git a/infra/stacks/observability/nixos/database-collector/host.nix b/infra/stacks/observability/nixos/database-collector/host.nix
new file mode 100644
index 0000000000..e51bb90d4b
--- /dev/null
+++ b/infra/stacks/observability/nixos/database-collector/host.nix
@@ -0,0 +1,98 @@
+{ lib, pkgs, ... }:
+{
+ system.stateVersion = "26.05";
+ image.baseName = "macro-database-collector";
+ virtualisation.diskSize = 8 * 1024;
+ ec2.efi = true;
+ networking.hostName = "database-collector-dev";
+ networking.firewall.enable = true;
+ services.openssh.enable = lib.mkForce false;
+ services.amazon-ssm-agent.enable = true;
+ virtualisation.amazon-init.enable = false;
+ systemd.services.fetch-ec2-metadata.enable = false;
+ systemd.services.print-host-key.enable = false;
+ environment.systemPackages = with pkgs; [
+ awscli2
+ curl
+ ];
+ environment.etc."ssl/certs/rds-us-east-1.pem".source = pkgs.fetchurl {
+ url = "https://truststore.pki.rds.amazonaws.com/us-east-1/us-east-1-bundle.pem";
+ hash = "sha256-sXEdErrlGDhYEoHiO2y5exB0AWhztNr8gO0UACRi3Xc=";
+ };
+ environment.etc."observability/prepare-volume.sh".source = ../../assets/prepare-volume.sh;
+ users.groups.alloy.gid = lib.mkForce 20005;
+ users.users.alloy = {
+ uid = lib.mkForce 20005;
+ isSystemUser = true;
+ group = "alloy";
+ createHome = lib.mkForce false;
+ };
+ services.alloy = {
+ enable = true;
+ configPath = pkgs.writeText "database.alloy" (builtins.readFile ./collector.alloy);
+ extraFlags = [
+ "--server.http.listen-addr=127.0.0.1:12345"
+ "--storage.path=/srv/observability/alloy"
+ "--disable-reporting"
+ ];
+ };
+ systemd.services.database-collector-bootstrap = {
+ description = "Load database credentials and mount retained metric WAL";
+ wantedBy = [ "multi-user.target" ];
+ wants = [ "network-online.target" ];
+ after = [ "network-online.target" ];
+ unitConfig.Upholds = [ "alloy.service" ];
+ startLimitIntervalSec = 0;
+ path = with pkgs; [
+ awscli2
+ bash
+ coreutils
+ e2fsprogs
+ util-linux
+ ];
+ script = ''
+ ${pkgs.python3}/bin/python3 ${./bootstrap.py}
+ ${pkgs.bash}/bin/bash ${../mount-data.sh}
+ install -d -m 0700 -o alloy -g alloy /srv/observability/alloy
+ '';
+ serviceConfig = {
+ Type = "oneshot";
+ RemainAfterExit = true;
+ Restart = "on-failure";
+ RestartMode = "direct";
+ RestartSec = 30;
+ TimeoutStartSec = 900;
+ UMask = "0077";
+ };
+ };
+ systemd.services.alloy = {
+ requires = [ "database-collector-bootstrap.service" ];
+ after = [ "database-collector-bootstrap.service" ];
+ partOf = [ "database-collector-bootstrap.service" ];
+ startLimitIntervalSec = 0;
+ serviceConfig = {
+ DynamicUser = lib.mkForce false;
+ User = "alloy";
+ Group = "alloy";
+ StateDirectory = lib.mkForce "";
+ WorkingDirectory = lib.mkForce "/srv/observability/alloy";
+ LoadCredential = map (name: "${name}:/run/macro-database-collector/${name}") [
+ "writer_dsn"
+ "replica_dsn"
+ "otlp_token"
+ ];
+ ExecStartPre = "${pkgs.util-linux}/bin/mountpoint -q /srv/observability";
+ Restart = lib.mkForce "always";
+ RestartMode = "direct";
+ RestartSec = lib.mkForce 30;
+ MemoryMax = "768M";
+ NoNewPrivileges = true;
+ ProtectSystem = "strict";
+ ProtectHome = true;
+ PrivateTmp = true;
+ ReadWritePaths = [ "/srv/observability/alloy" ];
+ IPAddressDeny = [ "169.254.169.254/32" ];
+ UMask = lib.mkForce "0077";
+ };
+ };
+}
diff --git a/infra/stacks/observability/nixos/database-dashboard.nix b/infra/stacks/observability/nixos/database-dashboard.nix
new file mode 100644
index 0000000000..ae073d67f9
--- /dev/null
+++ b/infra/stacks/observability/nixos/database-dashboard.nix
@@ -0,0 +1,124 @@
+{ lib, pkgs, ... }:
+let
+ datasource = {
+ type = "prometheus";
+ uid = "prometheus";
+ };
+ selector = ''job="integrations/postgres",deployment_environment="dev",db_instance=~"$database"'';
+ panel = id: title: expr: unit: {
+ inherit id title datasource;
+ type = "timeseries";
+ gridPos = {
+ x = lib.mod (id - 1) 2 * 12;
+ y = builtins.div (id - 1) 2 * 8;
+ w = 12;
+ h = 8;
+ };
+ targets = [
+ {
+ refId = "A";
+ inherit expr;
+ legendFormat = "{{db_instance}} {{datname}} {{mode}}";
+ }
+ ];
+ fieldConfig.defaults = { inherit unit; };
+ };
+ dashboard = {
+ uid = "macro-postgres";
+ title = "Macro PostgreSQL";
+ schemaVersion = 39;
+ version = 1;
+ editable = false;
+ tags = [
+ "macro"
+ "postgres"
+ ];
+ timezone = "browser";
+ refresh = "1m";
+ time = {
+ from = "now-6h";
+ to = "now";
+ };
+ description = "Dev writer and replica collected directly from PostgreSQL. Start with connection health and collection errors. No query text or plans are collected in this baseline. Missing data means collection needs attention, not that the database is healthy.";
+ templating.list = [
+ {
+ name = "database";
+ label = "Database instance";
+ type = "custom";
+ query = "macro-db-dev,macro-db-dev-read-replica";
+ includeAll = true;
+ allValue = "macro-db-dev.*";
+ multi = false;
+ current = {
+ text = "All";
+ value = "$__all";
+ };
+ options = [ ];
+ }
+ ];
+ links = [
+ {
+ title = "AWS database health";
+ type = "link";
+ url = "/d/macro-database";
+ }
+ ];
+ panels = [
+ (panel 1 "Database connection healthy (1 = yes)" "pg_up{${selector}}" "short")
+ (panel 2 "Exporter collection errors (1 = error)" "pg_exporter_last_scrape_error{${selector}}"
+ "short"
+ )
+ (panel 3 "Database connections" ''pg_stat_database_numbackends{${selector},datname="macrodb"}''
+ "short"
+ )
+ (panel 4 "Transactions per second"
+ ''sum by (db_instance, datname) (rate(pg_stat_database_xact_commit{${selector},datname="macrodb"}[5m]) + rate(pg_stat_database_xact_rollback{${selector},datname="macrodb"}[5m]))''
+ "ops"
+ )
+ (panel 5 "Deadlocks in five minutes"
+ ''increase(pg_stat_database_deadlocks{${selector},datname="macrodb"}[5m])''
+ "short"
+ )
+ (panel 6 "Temporary data written per second"
+ ''rate(pg_stat_database_temp_bytes{${selector},datname="macrodb"}[5m])''
+ "Bps"
+ )
+ (panel 7 "Locks by mode"
+ ''sum by (db_instance, mode) (pg_locks_count{${selector},datname="macrodb"})''
+ "short"
+ )
+ (panel 8 "Dead tuples (vacuum backlog indicator)"
+ ''sum by (db_instance) (pg_stat_user_tables_n_dead_tup{${selector},datname="macrodb"})''
+ "short"
+ )
+ (panel 9 "Database scrape freshness" "time() - timestamp(pg_up{${selector}})" "s")
+ (panel 10 "Remote-write pending samples"
+ ''prometheus_remote_storage_samples_pending{job="observability/database-collector",deployment_environment="dev"}''
+ "short"
+ )
+ (panel 11 "Scrape target healthy (1 = yes)" "up{${selector}}" "short")
+ (panel 12 "Remote-write failed samples per second"
+ ''rate(prometheus_remote_storage_samples_failed_total{job="observability/database-collector",deployment_environment="dev"}[5m])''
+ "ops"
+ )
+ ];
+ };
+in
+{
+ services.grafana.provision.dashboards.settings.providers = [
+ {
+ name = "macro-postgres";
+ orgId = 1;
+ folder = "Macro";
+ type = "file";
+ disableDeletion = true;
+ allowUiUpdates = false;
+ options.path = pkgs.linkFarm "macro-postgres-dashboard" [
+ {
+ name = "postgres.json";
+ path = pkgs.writeText "postgres.json" (builtins.toJSON dashboard);
+ }
+ ];
+ }
+ ];
+}
diff --git a/infra/stacks/observability/nixos/host.nix b/infra/stacks/observability/nixos/host.nix
index bf05e75039..5be83e4920 100644
--- a/infra/stacks/observability/nixos/host.nix
+++ b/infra/stacks/observability/nixos/host.nix
@@ -20,6 +20,7 @@ let
secretConsumers = [
"grafana.service"
"alloy-ingest.service"
+ "nginx.service"
];
dataUsers = {
grafana = 20001;
@@ -127,7 +128,10 @@ in
requires = [ "observability-bootstrap.service" ];
after = [ "observability-bootstrap.service" ];
partOf = [ "observability-bootstrap.service" ];
- path = [ pkgs.awscli2 ];
+ path = [
+ pkgs.awscli2
+ pkgs.openssl
+ ];
startLimitIntervalSec = 0;
unitConfig.Upholds = secretConsumers;
serviceConfig = {
diff --git a/infra/stacks/observability/nixos/proxy.nix b/infra/stacks/observability/nixos/proxy.nix
index ccb0be06cd..3f425a5b17 100644
--- a/infra/stacks/observability/nixos/proxy.nix
+++ b/infra/stacks/observability/nixos/proxy.nix
@@ -65,6 +65,18 @@ in
proxy_read_timeout 30s;
'';
};
+ "= /api/v1/write" = {
+ proxyPass = "http://127.0.0.1:9090/api/v1/write";
+ basicAuthFile = "/run/credentials/nginx.service/metrics.htpasswd";
+ extraConfig = ''
+ if ($observability_host != otlp) { return 404; }
+ limit_except POST { deny all; }
+ limit_req zone=ingest burst=200 nodelay;
+ limit_req_status 429;
+ proxy_read_timeout 30s;
+ proxy_set_header Authorization "";
+ '';
+ };
};
};
# Grafana Viewers can choose arbitrary datasource proxy paths. Allow query
diff --git a/infra/stacks/observability/nixos/publish-image.py b/infra/stacks/observability/nixos/publish-image.py
index f62484b66c..42334dbaff 100644
--- a/infra/stacks/observability/nixos/publish-image.py
+++ b/infra/stacks/observability/nixos/publish-image.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Publish a built image in Ohio. Explicit operator action; never run by Pulumi."""
+"""Publish a built image in its deployment region. Explicit operator action; never run by Pulumi."""
import argparse
import hashlib
import json
@@ -9,13 +9,15 @@ import time
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('image_directory', type=Path)
-parser.add_argument('bucket', help='Existing private Ohio image-artifact bucket')
+parser.add_argument('bucket', help='Existing private image-artifact bucket in the target region')
parser.add_argument('--role', default='vmimport', help='Existing VM Import service role')
+parser.add_argument('--region', choices=['us-east-1', 'us-east-2'], default='us-east-2')
+parser.add_argument('--name-prefix', choices=['macro-observability', 'macro-database-collector'], default='macro-observability')
args = parser.parse_args()
def aws(*arguments):
result = subprocess.run(
- ['aws', '--region', 'us-east-2', '--output', 'json', *arguments],
+ ['aws', '--region', args.region, '--output', 'json', *arguments],
check=True, capture_output=True, text=True,
)
return json.loads(result.stdout) if result.stdout.strip() else {}
@@ -23,8 +25,9 @@ def aws(*arguments):
identity = aws('sts', 'get-caller-identity')
if identity['Account'] != '569036502058':
raise SystemExit('Use the Macro AWS account (569036502058)')
-if aws('s3api', 'get-bucket-location', '--bucket', args.bucket)['LocationConstraint'] != 'us-east-2':
- raise SystemExit('Image bucket must be in Ohio')
+bucket_region = aws('s3api', 'get-bucket-location', '--bucket', args.bucket)['LocationConstraint'] or 'us-east-1'
+if bucket_region != args.region:
+ raise SystemExit('Image bucket must be in the target region')
public_access = aws('s3api', 'get-public-access-block', '--bucket', args.bucket)['PublicAccessBlockConfiguration']
if not all(public_access.get(key) for key in [
'BlockPublicAcls', 'IgnorePublicAcls', 'BlockPublicPolicy', 'RestrictPublicBuckets',
@@ -38,11 +41,11 @@ if image.suffix != '.vhd':
raise SystemExit('Expected the VHD image format')
with image.open('rb') as source:
digest = hashlib.file_digest(source, 'sha256').hexdigest()
-key = f'observability/{digest}.vhd'
-subprocess.run(['aws', '--region', 'us-east-2', 's3', 'cp', str(image),
+key = f'{args.name_prefix}/{digest}.vhd'
+subprocess.run(['aws', '--region', args.region, 's3', 'cp', str(image),
f's3://{args.bucket}/{key}', '--sse', 'AES256'], check=True)
task = aws('ec2', 'import-snapshot', '--role-name', args.role, '--encrypted',
- '--description', f'Macro observability {digest}',
+ '--description', f'{args.name_prefix} {digest}',
'--disk-container', json.dumps({
'Format': 'VHD', 'UserBucket': {'S3Bucket': args.bucket, 'S3Key': key},
}))['ImportTaskId']
@@ -60,7 +63,7 @@ while time.monotonic() < deadline:
else:
raise SystemExit(f'Import still pending: inspect {task} before retrying publication')
print(f'Imported snapshot: {snapshot}', flush=True)
-ami = aws('ec2', 'register-image', '--name', f'macro-observability-{digest[:20]}',
+ami = aws('ec2', 'register-image', '--name', f'{args.name_prefix}-{digest[:20]}',
'--architecture', 'x86_64', '--virtualization-type', 'hvm',
'--boot-mode', 'uefi', '--ena-support', '--imds-support', 'v2.0',
'--root-device-name', '/dev/xvda',
diff --git a/infra/stacks/observability/nixos/services.nix b/infra/stacks/observability/nixos/services.nix
index ed0f28e8b3..beb9485b71 100644
--- a/infra/stacks/observability/nixos/services.nix
+++ b/infra/stacks/observability/nixos/services.nix
@@ -17,6 +17,7 @@ in
imports = [
./dashboards.nix
./ci-dashboard.nix
+ ./database-dashboard.nix
./alerting.nix
];
services.grafana = {
@@ -57,6 +58,9 @@ in
];
};
systemd.services = {
+ nginx = needsSecrets // {
+ serviceConfig.LoadCredential = [ (credential "metrics.htpasswd") ];
+ };
grafana = needsSecrets // {
serviceConfig = {
EnvironmentFile = "/opt/observability/runtime.env";