Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Enable Grafana Drilldown and per-service exploration with Nix (#7882)
Wolf Mermelsteina6f031851309
diff --git a/infra/stacks/observability/Pulumi.prod.yaml b/infra/stacks/observability/Pulumi.prod.yamlindex f7cb6e2089..c0379a62ba 100644--- a/infra/stacks/observability/Pulumi.prod.yaml+++ b/infra/stacks/observability/Pulumi.prod.yaml@@ -1,5 +1,5 @@config:aws:region: us-east-2observability:alarmTopicArn: arn:aws:sns:us-east-2:569036502058:observability-prod-alerts- observability:amiId: ami-0a714f7bc630c0e2d+ observability:amiId: ami-082c5befa76bfc246observability:secretArn: arn:aws:secretsmanager:us-east-2:569036502058:secret:observability/prod-TwQpBJdiff --git a/infra/stacks/observability/README.md b/infra/stacks/observability/README.mdindex b53d101149..1d8148acb6 100644--- a/infra/stacks/observability/README.md+++ b/infra/stacks/observability/README.md@@ -326,6 +326,37 @@ still receives the authoritative copy.## Dashboards and alert migration+### Service exploration++Nix installs the signed Logs, Metrics, and Traces Drilldown apps through+`services.grafana.declarativePlugins`. Their versions and download hashes come+from the pinned `flake.lock`; no plugin download or update happens at boot.+Use **Drilldown → Logs** to choose a service and inspect its log volume, fields,+and patterns, or **Drilldown → Traces** to filter services, errors, and latency.+The dev telemetry dashboard also has a service selector and links to all three+Drilldown apps. The service selector discovers names from dev logs; a service+that emits only traces or metrics can be found in the corresponding Drilldown.++Loki enables pattern ingestion, log-level detection, and volume queries. Tempo's+metrics generator enables TraceQL metrics, span metrics, and service graphs.+Generator WAL and local blocks use the retained data volume; metrics blocks flush+to the existing trace bucket, and generated metrics go to local Prometheus.+These metrics start accumulating after this image is deployed. They describe+received spans, so sampling and missing instrumentation can prevent parity with+Datadog request counts. Span names remain in Tempo rather than becoming unbounded+Prometheus labels. Generated series are capped at 50,000; monitor+`tempo_metrics_generator_registry_series_limited_total` and local-block discards+before expanding ingestion. The existing Tempo memory limit remains in force.++Tempo spans link to the service's logs in a time window and its generated metrics;+logs carrying an OTLP trace ID link back to the trace. FireLens logs do not always+carry trace IDs, so service/time correlation is not a claim of exact trace-to-log+matching. Service graph edges require correctly related client/server spans.+Profiles, database query monitoring, RUM/session replay, and complete Datadog+dashboard/monitor migration remain separate coverage work.++### Provisioned dashboards and alerts+Nix provisions the `Macro` dashboard folder, a dev telemetry dashboard, and aCloudWatch database/Lambda dashboard. CloudWatch reads native metrics in Virginia;it does not replicate their history to Ohio, so these panels depend on Virginiadiff --git a/infra/stacks/observability/nixos/backend-config.nix b/infra/stacks/observability/nixos/backend-config.nixindex c31221baac..68080576ec 100644--- a/infra/stacks/observability/nixos/backend-config.nix+++ b/infra/stacks/observability/nixos/backend-config.nix@@ -39,6 +39,7 @@enabled = true;dir = "/srv/observability/loki/wal";};+ pattern_ingester.enabled = true;compactor = {working_directory = "/srv/observability/loki/compactor";retention_enabled = true;@@ -47,6 +48,8 @@limits_config = {retention_period = "720h";allow_structured_metadata = true;+ volume_enabled = true;+ discover_log_levels = true;ingestion_rate_mb = 8;ingestion_burst_size_mb = 16;max_query_parallelism = 4;@@ -68,6 +71,43 @@lifecycler.address = "127.0.0.1";};compactor.compaction.block_retention = "168h";+ metrics_generator = {+ ring.instance_addr = "127.0.0.1";+ processor = {+ service_graphs.dimensions = [ "deployment.environment" ];+ local_blocks = {+ filter_server_spans = false;+ flush_to_storage = true;+ max_live_traces = 10000;+ max_block_bytes = 50000000;+ concurrent_blocks = 2;+ };+ span_metrics = {+ # Keep unbounded span names out of Prometheus; they remain searchable in Tempo.+ intrinsic_dimensions.span_name = false;+ dimensions = [ "deployment.environment" ];+ };+ };+ traces_storage.path = "/srv/observability/tempo/generator/traces";+ storage = {+ path = "/srv/observability/tempo/generator/wal";+ remote_write = [+ {+ url = "http://127.0.0.1:9090/api/v1/write";+ send_exemplars = true;+ }+ ];+ };+ };+ overrides.defaults.metrics_generator = {+ processors = [+ "local-blocks"+ "span-metrics"+ "service-graphs"+ ];+ max_active_series = 50000;+ };+ query_frontend.metrics.concurrent_jobs = 4;storage.trace = {backend = "s3";wal.path = "/srv/observability/tempo/wal";diff --git a/infra/stacks/observability/nixos/dashboards.nix b/infra/stacks/observability/nixos/dashboards.nixindex 55bce0595e..e063ff5022 100644--- a/infra/stacks/observability/nixos/dashboards.nix+++ b/infra/stacks/observability/nixos/dashboards.nix@@ -56,49 +56,88 @@ let};};# ECS uses "dev"; browser and sync telemetry use "development".- applications = dashboard "macro-dev-telemetry" "Macro dev telemetry" [- (panel 1 "Logs per second by service" loki [- {- refId = "A";- expr = ''sum by (service_name) (rate({deployment_environment=~"dev|development"}[1m]))'';- }- ] "ops")- (panel 2 "Error logs in five minutes" loki [- {- refId = "A";- expr = ''sum by (service_name) (count_over_time({deployment_environment=~"dev|development"} | json parsed_level="level" | drop __error__, __error_details__ | parsed_level=~"(?i)error|fatal|critical" or severity_text=~"(?i)error|fatal|critical" [5m]))'';- }- ] "short")- (panel 3 "Task CPU usage" prometheus [- {- refId = "A";- expr = ''sum by (job) (ecs_task_cpu_usage_vcpu_vCPU{deployment_environment=~"dev|development"})'';- }- ] "short")- (panel 4 "Task memory usage" prometheus [- {- refId = "A";- expr = ''sum by (job) (ecs_task_memory_usage_Bytes{deployment_environment=~"dev|development"})'';- }- ] "bytes")- (- (panel 5 "Recent dev logs" loki [+ applications =+ (dashboard "macro-dev-telemetry" "Macro dev telemetry" [+ (panel 1 "Logs per second by service" loki [{refId = "A";- expr = ''{deployment_environment=~"dev|development"}'';+ expr = ''sum by (service_name) (rate({deployment_environment=~"dev|development", service_name=~"$service"}[1m]))'';+ }+ ] "ops")+ (panel 2 "Error logs in five minutes" loki [+ {+ refId = "A";+ expr = ''sum by (service_name) (count_over_time({deployment_environment=~"dev|development", service_name=~"$service"} | json parsed_level="level" | drop __error__, __error_details__ | parsed_level=~"(?i)error|fatal|critical" or severity_text=~"(?i)error|fatal|critical" [5m]))'';}] "short")- // {- type = "logs";- gridPos = {- x = 0;- y = 16;- w = 24;- h = 14;- };- }- )- ];+ (panel 3 "Task CPU usage" prometheus [+ {+ refId = "A";+ expr = ''sum by (job) (ecs_task_cpu_usage_vcpu_vCPU{deployment_environment=~"dev|development", job=~"$service"})'';+ }+ ] "short")+ (panel 4 "Task memory usage" prometheus [+ {+ refId = "A";+ expr = ''sum by (job) (ecs_task_memory_usage_Bytes{deployment_environment=~"dev|development", job=~"$service"})'';+ }+ ] "bytes")+ (+ (panel 5 "Recent dev logs" loki [+ {+ refId = "A";+ expr = ''{deployment_environment=~"dev|development", service_name=~"$service"}'';+ }+ ] "short")+ // {+ type = "logs";+ gridPos = {+ x = 0;+ y = 16;+ w = 24;+ h = 14;+ };+ }+ )+ ])+ // {+ description = "Choose a dev service to inspect its logs and ECS resources. Use Drilldown for log patterns, trace latency/errors, and metric exploration. Trace-to-log links show service logs around the span; individual logs link to traces when they carry a trace ID.";+ templating.list = [+ {+ name = "service";+ label = "Service";+ type = "query";+ datasource = loki;+ query = "label_values({deployment_environment=~\"dev|development\"}, service_name)";+ refresh = 2;+ sort = 1;+ includeAll = true;+ allValue = ".+";+ multi = false;+ current = {+ text = "All";+ value = "$__all";+ };+ }+ ];+ links = [+ {+ title = "Logs Drilldown";+ type = "link";+ url = "/a/grafana-lokiexplore-app/explore?var-ds=loki&var-primary_label=service_name%7C%3D~%7C\${service:percentencode}";+ }+ {+ title = "Traces Drilldown";+ type = "link";+ url = "/a/grafana-exploretraces-app/explore?var-ds=tempo&var-primarySignal=true&var-filters=resource.service.name%7C%3D~%7C\${service:percentencode}";+ }+ {+ title = "Metrics Drilldown";+ type = "link";+ url = "/a/grafana-metricsdrilldown-app/drilldown?var-ds=prometheus";+ }+ ];+ };database =(dashboard "macro-database" "Macro database and Lambda health" [(panel 1 "Database CPU" cloudwatch [diff --git a/infra/stacks/observability/nixos/datasources.nix b/infra/stacks/observability/nixos/datasources.nixindex d766dabc2a..ffc39070d3 100644--- a/infra/stacks/observability/nixos/datasources.nix+++ b/infra/stacks/observability/nixos/datasources.nix@@ -20,6 +20,15 @@url = "http://127.0.0.1:8081/prometheus";editable = false;isDefault = true;+ jsonData = {+ timeInterval = "30s";+ exemplarTraceIdDestinations = [+ {+ name = "traceID";+ datasourceUid = "tempo";+ }+ ];+ };}{name = "Loki";@@ -28,6 +37,16 @@access = "proxy";url = "http://127.0.0.1:8081/loki";editable = false;+ jsonData.derivedFields = [+ {+ name = "trace_id";+ matcherType = "label";+ matcherRegex = "trace_id";+ datasourceUid = "tempo";+ url = "$\${__value.raw}";+ urlDisplayLabel = "View trace";+ }+ ];}{name = "Tempo";@@ -36,6 +55,43 @@access = "proxy";url = "http://127.0.0.1:8081/tempo";editable = false;+ jsonData = {+ serviceMap.datasourceUid = "prometheus";+ nodeGraph.enabled = true;+ tracesToLogsV2 = {+ datasourceUid = "loki";+ spanStartTimeShift = "-1m";+ spanEndTimeShift = "1m";+ tags = [+ {+ key = "service.name";+ value = "service_name";+ }+ {+ key = "deployment.environment";+ value = "deployment_environment";+ }+ ];+ # FireLens logs do not always have a trace ID; show the service's logs in the span window.+ filterByTraceID = false;+ filterBySpanID = false;+ };+ tracesToMetrics = {+ datasourceUid = "prometheus";+ tags = [+ {+ key = "service.name";+ value = "service";+ }+ ];+ queries = [+ {+ name = "Recorded span rate";+ query = "sum(rate(traces_spanmetrics_calls_total{$\${__tags}}[5m]))";+ }+ ];+ };+ };}];}diff --git a/infra/stacks/observability/nixos/proxy.nix b/infra/stacks/observability/nixos/proxy.nixindex bda092d357..ccb0be06cd 100644--- a/infra/stacks/observability/nixos/proxy.nix+++ b/infra/stacks/observability/nixos/proxy.nix@@ -83,6 +83,15 @@ in"~ ^/prometheus/api/v1/(metadata|status/buildinfo|status/config|rules|alerts|targets|targets/metadata)$" =queryLocation "127.0.0.1:9090" "GET";"~ ^/loki/loki/api/v1/(query|query_range)$" = queryLocation "127.0.0.1:3100" "GET POST";+ "= /loki/loki/api/v1/drilldown-limits" =+ let+ location = queryLocation "127.0.0.1:3100" "GET";+ in+ location+ // {+ # Loki's limits handler requires the single-tenant ID even with auth disabled.+ extraConfig = location.extraConfig + "proxy_set_header X-Scope-OrgID fake;";+ };"~ ^/loki/loki/api/v1/(labels|label/[^/]+/values|series|index/stats|index/volume|index/volume_range|patterns|tail|status/buildinfo|format_query|detected_fields|detected_labels|detected_field/[^/]+/values)$" =letlocation = queryLocation "127.0.0.1:3100" "GET";diff --git a/infra/stacks/observability/nixos/services.nix b/infra/stacks/observability/nixos/services.nixindex 489b5624ac..3d05caf261 100644--- a/infra/stacks/observability/nixos/services.nix+++ b/infra/stacks/observability/nixos/services.nix@@ -20,6 +20,11 @@ in];services.grafana = {enable = true;+ declarativePlugins = with pkgs.grafanaPlugins; [+ grafana-lokiexplore-app+ grafana-metricsdrilldown-app+ grafana-exploretraces-app+ ];dataDir = "/srv/observability/grafana";settings = import ./grafana.nix;provision = {@@ -47,6 +52,7 @@ inextraFlags = ["--storage.tsdb.retention.size=100GB""--web.enable-remote-write-receiver"+ "--enable-feature=exemplar-storage"];};systemd.services = {