Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Tag metric per pod (#889)
Abel Dantas72e9900b8a91
diff --git a/core/switch_core/observability/bootstrap.py b/core/switch_core/observability/bootstrap.pyindex 7b3269cca..afa713ab1 100644--- a/core/switch_core/observability/bootstrap.py+++ b/core/switch_core/observability/bootstrap.py@@ -10,6 +10,7 @@ from __future__ import annotationsimport asyncioimport logging+import socketfrom collections.abc import Callable, Iterator, Mappingfrom dataclasses import dataclass@@ -252,6 +253,7 @@ def start_observability(commit_sha=switch_core_commit(),repository_url=switch_core_repository_url(),instance_id=process_id,+ pod_name=config.pod_name or socket.gethostname(),)if config.otlp_metrics_enabled:diff --git a/core/switch_core/observability/otlp.py b/core/switch_core/observability/otlp.pyindex 67a3142fd..6c9876b12 100644--- a/core/switch_core/observability/otlp.py+++ b/core/switch_core/observability/otlp.py@@ -142,6 +142,11 @@ class OtlpResource:several pods reports one series per pod rather than one series every podoverwrites. ``None`` for a resource that describes no single process, asproduct telemetry's does.++ ``pod_name`` is the pod (or host) the process runs on, the same across its+ restarts, which is what metric points are tagged with: the process id+ changes on every start, so tagging with it would add a new set of series+ each time."""service_name: str@@ -151,6 +156,7 @@ class OtlpResource:commit_sha: str | Nonerepository_url: str | Noneinstance_id: str | None+ pod_name: str | None = Nonedef attributes(self) -> dict[str, AttributeValue]:values: dict[str, AttributeValue] = {@@ -202,11 +208,27 @@ class MetricPayload:histograms: Sequence[HistogramPoint]+def _point_attributes(+ attributes: Mapping[str, AttributeValue], extra: Mapping[str, AttributeValue]+) -> list[dict[str, Any]]:+ """A point's own attributes plus the ones every point carries, refusing a+ metric that already uses one of those names for something else."""+ clash = attributes.keys() & extra.keys()+ if clash:+ raise ValueError(+ f"Metric attribute(s) {sorted(clash)} are reserved for the process tags."+ )+ return otlp_attributes({**attributes, **extra})++def _number_data_point(- point: NumberPoint, start_nanos: int, end_nanos: int+ point: NumberPoint,+ start_nanos: int,+ end_nanos: int,+ extra: Mapping[str, AttributeValue],) -> dict[str, Any]:return {- "attributes": otlp_attributes(point.attributes),+ "attributes": _point_attributes(point.attributes, extra),"startTimeUnixNano": str(start_nanos),"timeUnixNano": str(end_nanos),# `asDouble`, not `asInt`: int64 is a JSON string, which invites a@@ -216,10 +238,13 @@ def _number_data_point(def _histogram_data_point(- point: HistogramPoint, start_nanos: int, end_nanos: int+ point: HistogramPoint,+ start_nanos: int,+ end_nanos: int,+ extra: Mapping[str, AttributeValue],) -> dict[str, Any]:return {- "attributes": otlp_attributes(point.attributes),+ "attributes": _point_attributes(point.attributes, extra),"startTimeUnixNano": str(start_nanos),"timeUnixNano": str(end_nanos),# fixed64 on the wire, so strings in JSON.@@ -231,7 +256,10 @@ def _histogram_data_point(def _encode_metric(- metric: MetricPayload, start_nanos: int, end_nanos: int+ metric: MetricPayload,+ start_nanos: int,+ end_nanos: int,+ extra: Mapping[str, AttributeValue],) -> dict[str, Any]:encoded: dict[str, Any] = {"name": metric.name,@@ -243,7 +271,7 @@ def _encode_metric(encoded["gauge"] = {"dataPoints": [{- "attributes": otlp_attributes(point.attributes),+ "attributes": _point_attributes(point.attributes, extra),"timeUnixNano": str(end_nanos),"asDouble": point.value,}@@ -253,7 +281,7 @@ def _encode_metric(elif metric.kind == "sum":encoded["sum"] = {"dataPoints": [- _number_data_point(point, start_nanos, end_nanos)+ _number_data_point(point, start_nanos, end_nanos, extra)for point in metric.numbers],"aggregationTemporality": AGGREGATION_TEMPORALITY_DELTA,@@ -262,7 +290,7 @@ def _encode_metric(elif metric.kind == "histogram":encoded["histogram"] = {"dataPoints": [- _histogram_data_point(point, start_nanos, end_nanos)+ _histogram_data_point(point, start_nanos, end_nanos, extra)for point in metric.histograms],"aggregationTemporality": AGGREGATION_TEMPORALITY_DELTA,@@ -278,7 +306,16 @@ def build_metrics_payload(start_nanos: int,end_nanos: int,) -> dict[str, Any]:- """One export interval's metrics as a single OTLP request body."""+ """One export interval's metrics as a single OTLP request body.++ Every point also carries its pod as `pod_name`. The resource names the+ process (`service.instance.id`), but Datadog does not make resource+ attributes tags, so without it the pods of one deployment write the same+ series and each gauge shows whichever pod reported last.+ """+ extra: dict[str, AttributeValue] = (+ {"pod_name": resource.pod_name} if resource.pod_name else {}+ )return {"resourceMetrics": [{@@ -294,7 +331,7 @@ def build_metrics_payload(),},"metrics": [- _encode_metric(metric, start_nanos, end_nanos)+ _encode_metric(metric, start_nanos, end_nanos, extra)for metric in metrics],}diff --git a/core/tests/switch_core/observability/test_otlp.py b/core/tests/switch_core/observability/test_otlp.pyindex f9e6d1ea9..c3bb2b50f 100644--- a/core/tests/switch_core/observability/test_otlp.py+++ b/core/tests/switch_core/observability/test_otlp.py@@ -107,6 +107,93 @@ def test_resource_names_the_process_so_pods_report_apart():assert resource.attributes()["service.instance.id"] == "switch-core-7d9f-abcd1234"+def test_every_point_names_its_pod_as_a_tag():+ """Datadog makes point attributes tags but not resource ones, so two pods'+ gauges would otherwise land in one series and overwrite each other. The+ pod, not the process: the process id changes on every restart."""+ resource = OtlpResource(+ service_name="switch-core",+ service_version="1.2.3",+ environment="pilot",+ deployment_id=RESOURCE.deployment_id,+ commit_sha=None,+ repository_url=None,+ instance_id="switch-core-7d9f-abcd1234",+ pod_name="switch-core-7d9f",+ )+ gauge = MetricPayload(+ name="switch.agents.connected",+ unit="{agent}",+ description="Agents connected.",+ kind="gauge",+ numbers=[NumberPoint(attributes={"transport": "websocket"}, value=7.0)],+ histograms=(),+ )+ histogram = MetricPayload(+ name="switch.db.pool.wait.duration",+ unit="ms",+ description="Pool waits.",+ kind="histogram",+ numbers=(),+ histograms=[+ HistogramPoint(+ attributes={},+ count=1,+ total=3.0,+ bucket_counts=[1, 0],+ bounds=[5.0],+ )+ ],+ )+ payload = build_metrics_payload(+ [gauge, _sum_metric(), histogram], resource, START_NANOS, END_NANOS+ )+ metrics = payload["resourceMetrics"][0]["scopeMetrics"][0]["metrics"]++ for points in (+ metrics[0]["gauge"]["dataPoints"],+ metrics[1]["sum"]["dataPoints"],+ metrics[2]["histogram"]["dataPoints"],+ ):+ tags = {a["key"]: a["value"] for a in points[0]["attributes"]}+ assert tags["pod_name"] == {"stringValue": "switch-core-7d9f"}+ gauge_tags = {a["key"] for a in metrics[0]["gauge"]["dataPoints"][0]["attributes"]}+ assert gauge_tags == {"transport", "pod_name"}+++def test_a_metric_attribute_cannot_take_the_pod_tag_s_name():+ resource = OtlpResource(+ service_name="switch-core",+ service_version=None,+ environment=None,+ deployment_id=RESOURCE.deployment_id,+ commit_sha=None,+ repository_url=None,+ instance_id="switch-core-7d9f-abcd1234",+ pod_name="switch-core-7d9f",+ )+ gauge = MetricPayload(+ name="switch.leases.held",+ unit="{lease}",+ description="Leases.",+ kind="gauge",+ numbers=[NumberPoint(attributes={"pod_name": "other"}, value=1.0)],+ histograms=(),+ )++ with pytest.raises(ValueError, match="pod_name"):+ build_metrics_payload([gauge], resource, START_NANOS, END_NANOS)+++def test_points_carry_no_pod_for_a_resource_without_one():+ payload = build_metrics_payload([_sum_metric()], RESOURCE, START_NANOS, END_NANOS)+ point = payload["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0]["sum"][+ "dataPoints"+ ][0]++ assert [a["key"] for a in point["attributes"]] == ["route"]++def test_resource_names_the_commit_it_was_built_from():resource = OtlpResource(service_name="switch-core",diff --git a/deploy/observability/README.md b/deploy/observability/README.mdindex 7e9a1f003..65690f766 100644--- a/deploy/observability/README.md+++ b/deploy/observability/README.md@@ -115,3 +115,16 @@ reporting the container's limit would make this alertable properly.deliberately unpanelled — it is a curiosity for a service like this one, not asignal. It is there for the day someone is chasing a memory question and wantsit.++## Across pods++`multi-pod-dashboard.json` shows switch-core per pod (`pod_name`), to see+whether work is shared between pods and whether anything is handled twice.+Gauges are split by pod; platform events, messages and the database server's+own counters show one total. Pick the deployment with `env` and `service`. It+needs a build that tags points with `pod_name`; earlier builds show one+unsplit series.++`dashboard.json` sums per-pod gauges (consumers, connectors, pool, memory) so+its totals stay totals with several pods, and the pool-saturation monitor+alerts per pod, since one pod can exhaust its pool while the others idle.diff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.jsonindex def5e4e20..4f68f5e04 100644--- a/deploy/observability/dashboard.json+++ b/deploy/observability/dashboard.json@@ -19,7 +19,7 @@{"definition": {"type": "note",- "content": "**Is it working?** Each dependency reports 1 when its last check passed and 0 when it failed. Only the database takes the server out of service \u2014 switch-core runs as a single replica, so failing readiness empties the service rather than shifting traffic. A crashed bridge is a real fault that is deliberately not fatal.",+ "content": "**Is it working?** Each dependency reports 1 when its last check passed and 0 when it failed. Only the database takes a pod out of service. With one replica that empties the service; with several, traffic moves to the pods that still pass. A crashed bridge is a real fault that is deliberately not fatal.","background_color": "blue","font_size": "14","text_align": "left",@@ -111,7 +111,7 @@"title": "Consumers running (read loops)","requests": [{- "q": "avg:switch.consumers.running{$env,$service}",+ "q": "sum:switch.consumers.running{$env,$service}","aggregator": "last"}],@@ -458,15 +458,15 @@"show_legend": true,"requests": [{- "q": "max:switch.db.pool.in_use{$env,$service}",+ "q": "sum:switch.db.pool.in_use{$env,$service}","display_type": "area"},{- "q": "avg:switch.db.pool.size{$env,$service}",+ "q": "sum:switch.db.pool.size{$env,$service}","display_type": "line"},{- "q": "avg:switch.db.pool.overflow{$env,$service}",+ "q": "sum:switch.db.pool.overflow{$env,$service}","display_type": "line"}]@@ -509,7 +509,7 @@"show_legend": false,"requests": [{- "q": "avg:switch.runtime.memory_rss{$env,$service}",+ "q": "sum:switch.runtime.memory_rss{$env,$service}","display_type": "area"}]@@ -547,7 +547,7 @@"show_legend": false,"requests": [{- "q": "avg:switch.runtime.open_fds{$env,$service}",+ "q": "sum:switch.runtime.open_fds{$env,$service}","display_type": "line"}]@@ -672,7 +672,7 @@"show_legend": true,"requests": [{- "q": "avg:switch.connectors.running{$env,$service}",+ "q": "sum:switch.connectors.running{$env,$service}","display_type": "line"}]@@ -1106,11 +1106,11 @@"show_legend": true,"requests": [{- "q": "max:switch.delivery_cache.bytes{$env,$service}",+ "q": "sum:switch.delivery_cache.bytes{$env,$service}","display_type": "line"},{- "q": "max:switch.delivery_cache.rooms{$env,$service}",+ "q": "sum:switch.delivery_cache.rooms{$env,$service}","display_type": "line"}]diff --git a/deploy/observability/monitors.json b/deploy/observability/monitors.jsonindex 2092da7c2..972fcef0e 100644--- a/deploy/observability/monitors.json+++ b/deploy/observability/monitors.json@@ -131,8 +131,8 @@{"name": "[Switch] the database pool is nearly exhausted","type": "metric alert",- "query": "avg(last_10m):avg:switch.db.pool.in_use{service:switch-core} / avg:switch.db.pool.size{service:switch-core} > 0.9",- "message": "The connection pool has been at more than 90% of its nominal size for ten minutes.\n\nNominal, not total: the pool may open a few connections beyond its size before anything blocks, so this is an early warning rather than the moment things break. What happens if it keeps climbing is that requests start waiting for a connection rather than failing, so the symptom is latency everywhere and then a readiness check that times out \u2014 which reads as \"the database is down\" when the database is fine and this server is holding all of it.\n\nLook for a query holding a connection longer than it should before raising the pool size.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",+ "query": "avg(last_10m):avg:switch.db.pool.in_use{service:switch-core} by {pod_name} / avg:switch.db.pool.size{service:switch-core} by {pod_name} > 0.9",+ "message": "The connection pool of {{pod_name.name}} has been at more than 90% of its nominal size for ten minutes. Each pod has its own pool, so this alerts per pod.\n\nNominal, not total: the pool may open a few connections beyond its size before anything blocks, so this is an early warning rather than the moment things break. What happens if it keeps climbing is that requests start waiting for a connection rather than failing, so the symptom is latency everywhere and then a readiness check that times out \u2014 which reads as \"the database is down\" when the database is fine and this server is holding all of it.\n\nLook for a query holding a connection longer than it should before raising the pool size.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE","tags": ["service:switch-core","team:switch"diff --git a/deploy/observability/multi-pod-dashboard.json b/deploy/observability/multi-pod-dashboard.jsonnew file mode 100644index 000000000..1870b6cf4--- /dev/null+++ b/deploy/observability/multi-pod-dashboard.json@@ -0,0 +1,547 @@+{+ "title": "Switch — core across pods",+ "description": "switch-core per pod: is the work split, is anything doubled, how hot are the pods and the database. Source: deploy/observability/multi-pod-dashboard.json.",+ "layout_type": "ordered",+ "reflow_type": "fixed",+ "template_variables": [+ {+ "name": "env",+ "prefix": "env",+ "default": "*"+ },+ {+ "name": "service",+ "prefix": "service",+ "default": "switch-core"+ },+ {+ "name": "pod_name",+ "prefix": "pod_name",+ "default": "*"+ }+ ],+ "widgets": [+ {+ "definition": {+ "type": "note",+ "content": "**Is work split across the pods, and is nothing doubled?** Every series is per pod (`pod_name`). With two pods, bridges, agents and pool use should split between them and sum to what one pod carried; events in vs out should keep the one-pod ratio. Pick the deployment with `env` and `service`.",+ "background_color": "blue",+ "font_size": "14",+ "text_align": "left",+ "vertical_align": "top",+ "show_tick": false,+ "has_padding": true+ },+ "layout": {+ "x": 0,+ "y": 0,+ "width": 12,+ "height": 2+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Bridges running, by pod",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "sum:switch.bridges.running{$env,$service,$pod_name} by {pod_name}"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 0,+ "y": 2,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Agents connected, by pod",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "sum:switch.agents.connected{$env,$service,$pod_name} by {pod_name}"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 4,+ "y": 2,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Consumers and connectors running, by pod",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "sum:switch.consumers.running{$env,$service,$pod_name} by {pod_name}"+ },+ {+ "data_source": "metrics",+ "name": "q1",+ "query": "sum:switch.connectors.running{$env,$service,$pod_name} by {pod_name}"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ },+ {+ "formula": "q1"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 8,+ "y": 2,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Platform events in / out",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "sum:switch.bridge.events_in{$env,$service,$pod_name}.as_count()"+ },+ {+ "data_source": "metrics",+ "name": "q1",+ "query": "sum:switch.bridge.events_out{$env,$service,$pod_name}.as_count()"+ }+ ],+ "response_format": "timeseries",+ "display_type": "bars",+ "formulas": [+ {+ "formula": "q0"+ },+ {+ "formula": "q1"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 0,+ "y": 5,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Messages sent / delivered",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "sum:switch.messages.sent{$env,$service,$pod_name}.as_count()"+ },+ {+ "data_source": "metrics",+ "name": "q1",+ "query": "sum:switch.messages.delivered{$env,$service,$pod_name}.as_count()"+ }+ ],+ "response_format": "timeseries",+ "display_type": "bars",+ "formulas": [+ {+ "formula": "q0"+ },+ {+ "formula": "q1"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 4,+ "y": 5,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Delivery lag (max), by pod",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "max:switch.messages.delivery_lag{$env,$service,$pod_name} by {pod_name}"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 8,+ "y": 5,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "DB pool in use, by pod",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "max:switch.db.pool.in_use{$env,$service,$pod_name} by {pod_name}"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 0,+ "y": 8,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "DB pool wait (max), by pod",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "max:switch.db.pool.wait.duration{$env,$service,$pod_name} by {pod_name}"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 4,+ "y": 8,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "DB server connections",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "max:switch.db.server.connections{$env,$service}"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 8,+ "y": 8,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "CPU (cores), by pod",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "sum:switch.runtime.cpu_seconds{$env,$service,$pod_name} by {pod_name}.as_rate()"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 0,+ "y": 11,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Memory RSS, by pod",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "max:switch.runtime.memory_rss{$env,$service,$pod_name} by {pod_name}"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 4,+ "y": 11,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Event loop lag (max), by pod",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "max:switch.runtime.event_loop_lag{$env,$service,$pod_name} by {pod_name}"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 8,+ "y": 11,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Bridge errors, by pod",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "sum:switch.bridge.errors{$env,$service,$pod_name} by {pod_name}.as_count()"+ }+ ],+ "response_format": "timeseries",+ "display_type": "bars",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 0,+ "y": 14,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Database transactions",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "avg:switch.db.server.transactions{$env,$service}.as_rate()"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 4,+ "y": 14,+ "width": 4,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "DB query time (max), by pod",+ "show_legend": true,+ "legend_layout": "auto",+ "requests": [+ {+ "queries": [+ {+ "data_source": "metrics",+ "name": "q0",+ "query": "max:switch.db.query.duration{$env,$service,$pod_name} by {pod_name}"+ }+ ],+ "response_format": "timeseries",+ "display_type": "line",+ "formulas": [+ {+ "formula": "q0"+ }+ ]+ }+ ]+ },+ "layout": {+ "x": 8,+ "y": 14,+ "width": 4,+ "height": 3+ }+ }+ ]+}