Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Enable Grafana Drilldown and per-service exploration with Nix (#7882)

Wolf Mermelsteina6f031851309
diff --git a/infra/stacks/observability/Pulumi.prod.yaml b/infra/stacks/observability/Pulumi.prod.yaml
index f7cb6e2089..c0379a62ba 100644
--- a/infra/stacks/observability/Pulumi.prod.yaml
+++ b/infra/stacks/observability/Pulumi.prod.yaml
@@ -1,5 +1,5 @@
config:
aws:region: us-east-2
observability:alarmTopicArn: arn:aws:sns:us-east-2:569036502058:observability-prod-alerts
- observability:amiId: ami-0a714f7bc630c0e2d
+ observability:amiId: ami-082c5befa76bfc246
observability:secretArn: arn:aws:secretsmanager:us-east-2:569036502058:secret:observability/prod-TwQpBJ
diff --git a/infra/stacks/observability/README.md b/infra/stacks/observability/README.md
index b53d101149..1d8148acb6 100644
--- a/infra/stacks/observability/README.md
+++ b/infra/stacks/observability/README.md
@@ -326,6 +326,37 @@ still receives the authoritative copy.
## Dashboards and alert migration
+### Service exploration
+
+Nix installs the signed Logs, Metrics, and Traces Drilldown apps through
+`services.grafana.declarativePlugins`. Their versions and download hashes come
+from the pinned `flake.lock`; no plugin download or update happens at boot.
+Use **Drilldown → Logs** to choose a service and inspect its log volume, fields,
+and patterns, or **Drilldown → Traces** to filter services, errors, and latency.
+The dev telemetry dashboard also has a service selector and links to all three
+Drilldown apps. The service selector discovers names from dev logs; a service
+that emits only traces or metrics can be found in the corresponding Drilldown.
+
+Loki enables pattern ingestion, log-level detection, and volume queries. Tempo's
+metrics generator enables TraceQL metrics, span metrics, and service graphs.
+Generator WAL and local blocks use the retained data volume; metrics blocks flush
+to the existing trace bucket, and generated metrics go to local Prometheus.
+These metrics start accumulating after this image is deployed. They describe
+received spans, so sampling and missing instrumentation can prevent parity with
+Datadog request counts. Span names remain in Tempo rather than becoming unbounded
+Prometheus labels. Generated series are capped at 50,000; monitor
+`tempo_metrics_generator_registry_series_limited_total` and local-block discards
+before expanding ingestion. The existing Tempo memory limit remains in force.
+
+Tempo spans link to the service's logs in a time window and its generated metrics;
+logs carrying an OTLP trace ID link back to the trace. FireLens logs do not always
+carry trace IDs, so service/time correlation is not a claim of exact trace-to-log
+matching. Service graph edges require correctly related client/server spans.
+Profiles, database query monitoring, RUM/session replay, and complete Datadog
+dashboard/monitor migration remain separate coverage work.
+
+### Provisioned dashboards and alerts
+
Nix provisions the `Macro` dashboard folder, a dev telemetry dashboard, and a
CloudWatch database/Lambda dashboard. CloudWatch reads native metrics in Virginia;
it does not replicate their history to Ohio, so these panels depend on Virginia
diff --git a/infra/stacks/observability/nixos/backend-config.nix b/infra/stacks/observability/nixos/backend-config.nix
index c31221baac..68080576ec 100644
--- a/infra/stacks/observability/nixos/backend-config.nix
+++ b/infra/stacks/observability/nixos/backend-config.nix
@@ -39,6 +39,7 @@
enabled = true;
dir = "/srv/observability/loki/wal";
};
+ pattern_ingester.enabled = true;
compactor = {
working_directory = "/srv/observability/loki/compactor";
retention_enabled = true;
@@ -47,6 +48,8 @@
limits_config = {
retention_period = "720h";
allow_structured_metadata = true;
+ volume_enabled = true;
+ discover_log_levels = true;
ingestion_rate_mb = 8;
ingestion_burst_size_mb = 16;
max_query_parallelism = 4;
@@ -68,6 +71,43 @@
lifecycler.address = "127.0.0.1";
};
compactor.compaction.block_retention = "168h";
+ metrics_generator = {
+ ring.instance_addr = "127.0.0.1";
+ processor = {
+ service_graphs.dimensions = [ "deployment.environment" ];
+ local_blocks = {
+ filter_server_spans = false;
+ flush_to_storage = true;
+ max_live_traces = 10000;
+ max_block_bytes = 50000000;
+ concurrent_blocks = 2;
+ };
+ span_metrics = {
+ # Keep unbounded span names out of Prometheus; they remain searchable in Tempo.
+ intrinsic_dimensions.span_name = false;
+ dimensions = [ "deployment.environment" ];
+ };
+ };
+ traces_storage.path = "/srv/observability/tempo/generator/traces";
+ storage = {
+ path = "/srv/observability/tempo/generator/wal";
+ remote_write = [
+ {
+ url = "http://127.0.0.1:9090/api/v1/write";
+ send_exemplars = true;
+ }
+ ];
+ };
+ };
+ overrides.defaults.metrics_generator = {
+ processors = [
+ "local-blocks"
+ "span-metrics"
+ "service-graphs"
+ ];
+ max_active_series = 50000;
+ };
+ query_frontend.metrics.concurrent_jobs = 4;
storage.trace = {
backend = "s3";
wal.path = "/srv/observability/tempo/wal";
diff --git a/infra/stacks/observability/nixos/dashboards.nix b/infra/stacks/observability/nixos/dashboards.nix
index 55bce0595e..e063ff5022 100644
--- a/infra/stacks/observability/nixos/dashboards.nix
+++ b/infra/stacks/observability/nixos/dashboards.nix
@@ -56,49 +56,88 @@ let
};
};
# ECS uses "dev"; browser and sync telemetry use "development".
- applications = dashboard "macro-dev-telemetry" "Macro dev telemetry" [
- (panel 1 "Logs per second by service" loki [
- {
- refId = "A";
- expr = ''sum by (service_name) (rate({deployment_environment=~"dev|development"}[1m]))'';
- }
- ] "ops")
- (panel 2 "Error logs in five minutes" loki [
- {
- refId = "A";
- expr = ''sum by (service_name) (count_over_time({deployment_environment=~"dev|development"} | json parsed_level="level" | drop __error__, __error_details__ | parsed_level=~"(?i)error|fatal|critical" or severity_text=~"(?i)error|fatal|critical" [5m]))'';
- }
- ] "short")
- (panel 3 "Task CPU usage" prometheus [
- {
- refId = "A";
- expr = ''sum by (job) (ecs_task_cpu_usage_vcpu_vCPU{deployment_environment=~"dev|development"})'';
- }
- ] "short")
- (panel 4 "Task memory usage" prometheus [
- {
- refId = "A";
- expr = ''sum by (job) (ecs_task_memory_usage_Bytes{deployment_environment=~"dev|development"})'';
- }
- ] "bytes")
- (
- (panel 5 "Recent dev logs" loki [
+ applications =
+ (dashboard "macro-dev-telemetry" "Macro dev telemetry" [
+ (panel 1 "Logs per second by service" loki [
{
refId = "A";
- expr = ''{deployment_environment=~"dev|development"}'';
+ expr = ''sum by (service_name) (rate({deployment_environment=~"dev|development", service_name=~"$service"}[1m]))'';
+ }
+ ] "ops")
+ (panel 2 "Error logs in five minutes" loki [
+ {
+ refId = "A";
+ expr = ''sum by (service_name) (count_over_time({deployment_environment=~"dev|development", service_name=~"$service"} | json parsed_level="level" | drop __error__, __error_details__ | parsed_level=~"(?i)error|fatal|critical" or severity_text=~"(?i)error|fatal|critical" [5m]))'';
}
] "short")
- // {
- type = "logs";
- gridPos = {
- x = 0;
- y = 16;
- w = 24;
- h = 14;
- };
- }
- )
- ];
+ (panel 3 "Task CPU usage" prometheus [
+ {
+ refId = "A";
+ expr = ''sum by (job) (ecs_task_cpu_usage_vcpu_vCPU{deployment_environment=~"dev|development", job=~"$service"})'';
+ }
+ ] "short")
+ (panel 4 "Task memory usage" prometheus [
+ {
+ refId = "A";
+ expr = ''sum by (job) (ecs_task_memory_usage_Bytes{deployment_environment=~"dev|development", job=~"$service"})'';
+ }
+ ] "bytes")
+ (
+ (panel 5 "Recent dev logs" loki [
+ {
+ refId = "A";
+ expr = ''{deployment_environment=~"dev|development", service_name=~"$service"}'';
+ }
+ ] "short")
+ // {
+ type = "logs";
+ gridPos = {
+ x = 0;
+ y = 16;
+ w = 24;
+ h = 14;
+ };
+ }
+ )
+ ])
+ // {
+ description = "Choose a dev service to inspect its logs and ECS resources. Use Drilldown for log patterns, trace latency/errors, and metric exploration. Trace-to-log links show service logs around the span; individual logs link to traces when they carry a trace ID.";
+ templating.list = [
+ {
+ name = "service";
+ label = "Service";
+ type = "query";
+ datasource = loki;
+ query = "label_values({deployment_environment=~\"dev|development\"}, service_name)";
+ refresh = 2;
+ sort = 1;
+ includeAll = true;
+ allValue = ".+";
+ multi = false;
+ current = {
+ text = "All";
+ value = "$__all";
+ };
+ }
+ ];
+ links = [
+ {
+ title = "Logs Drilldown";
+ type = "link";
+ url = "/a/grafana-lokiexplore-app/explore?var-ds=loki&var-primary_label=service_name%7C%3D~%7C\${service:percentencode}";
+ }
+ {
+ title = "Traces Drilldown";
+ type = "link";
+ url = "/a/grafana-exploretraces-app/explore?var-ds=tempo&var-primarySignal=true&var-filters=resource.service.name%7C%3D~%7C\${service:percentencode}";
+ }
+ {
+ title = "Metrics Drilldown";
+ type = "link";
+ url = "/a/grafana-metricsdrilldown-app/drilldown?var-ds=prometheus";
+ }
+ ];
+ };
database =
(dashboard "macro-database" "Macro database and Lambda health" [
(panel 1 "Database CPU" cloudwatch [
diff --git a/infra/stacks/observability/nixos/datasources.nix b/infra/stacks/observability/nixos/datasources.nix
index d766dabc2a..ffc39070d3 100644
--- a/infra/stacks/observability/nixos/datasources.nix
+++ b/infra/stacks/observability/nixos/datasources.nix
@@ -20,6 +20,15 @@
url = "http://127.0.0.1:8081/prometheus";
editable = false;
isDefault = true;
+ jsonData = {
+ timeInterval = "30s";
+ exemplarTraceIdDestinations = [
+ {
+ name = "traceID";
+ datasourceUid = "tempo";
+ }
+ ];
+ };
}
{
name = "Loki";
@@ -28,6 +37,16 @@
access = "proxy";
url = "http://127.0.0.1:8081/loki";
editable = false;
+ jsonData.derivedFields = [
+ {
+ name = "trace_id";
+ matcherType = "label";
+ matcherRegex = "trace_id";
+ datasourceUid = "tempo";
+ url = "$\${__value.raw}";
+ urlDisplayLabel = "View trace";
+ }
+ ];
}
{
name = "Tempo";
@@ -36,6 +55,43 @@
access = "proxy";
url = "http://127.0.0.1:8081/tempo";
editable = false;
+ jsonData = {
+ serviceMap.datasourceUid = "prometheus";
+ nodeGraph.enabled = true;
+ tracesToLogsV2 = {
+ datasourceUid = "loki";
+ spanStartTimeShift = "-1m";
+ spanEndTimeShift = "1m";
+ tags = [
+ {
+ key = "service.name";
+ value = "service_name";
+ }
+ {
+ key = "deployment.environment";
+ value = "deployment_environment";
+ }
+ ];
+ # FireLens logs do not always have a trace ID; show the service's logs in the span window.
+ filterByTraceID = false;
+ filterBySpanID = false;
+ };
+ tracesToMetrics = {
+ datasourceUid = "prometheus";
+ tags = [
+ {
+ key = "service.name";
+ value = "service";
+ }
+ ];
+ queries = [
+ {
+ name = "Recorded span rate";
+ query = "sum(rate(traces_spanmetrics_calls_total{$\${__tags}}[5m]))";
+ }
+ ];
+ };
+ };
}
];
}
diff --git a/infra/stacks/observability/nixos/proxy.nix b/infra/stacks/observability/nixos/proxy.nix
index bda092d357..ccb0be06cd 100644
--- a/infra/stacks/observability/nixos/proxy.nix
+++ b/infra/stacks/observability/nixos/proxy.nix
@@ -83,6 +83,15 @@ in
"~ ^/prometheus/api/v1/(metadata|status/buildinfo|status/config|rules|alerts|targets|targets/metadata)$" =
queryLocation "127.0.0.1:9090" "GET";
"~ ^/loki/loki/api/v1/(query|query_range)$" = queryLocation "127.0.0.1:3100" "GET POST";
+ "= /loki/loki/api/v1/drilldown-limits" =
+ let
+ location = queryLocation "127.0.0.1:3100" "GET";
+ in
+ location
+ // {
+ # Loki's limits handler requires the single-tenant ID even with auth disabled.
+ extraConfig = location.extraConfig + "proxy_set_header X-Scope-OrgID fake;";
+ };
"~ ^/loki/loki/api/v1/(labels|label/[^/]+/values|series|index/stats|index/volume|index/volume_range|patterns|tail|status/buildinfo|format_query|detected_fields|detected_labels|detected_field/[^/]+/values)$" =
let
location = queryLocation "127.0.0.1:3100" "GET";
diff --git a/infra/stacks/observability/nixos/services.nix b/infra/stacks/observability/nixos/services.nix
index 489b5624ac..3d05caf261 100644
--- a/infra/stacks/observability/nixos/services.nix
+++ b/infra/stacks/observability/nixos/services.nix
@@ -20,6 +20,11 @@ in
];
services.grafana = {
enable = true;
+ declarativePlugins = with pkgs.grafanaPlugins; [
+ grafana-lokiexplore-app
+ grafana-metricsdrilldown-app
+ grafana-exploretraces-app
+ ];
dataDir = "/srv/observability/grafana";
settings = import ./grafana.nix;
provision = {
@@ -47,6 +52,7 @@ in
extraFlags = [
"--storage.tsdb.retention.size=100GB"
"--web.enable-remote-write-receiver"
+ "--enable-feature=exemplar-storage"
];
};
systemd.services = {