SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
metrics: overhaul Grafana dashboard for full metric coverage (#9956)
Chris Lu3fd5018bd28e
diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.jsonindex 67c2fea138..d8df871901 100644--- a/other/metrics/grafana_seaweedfs.json+++ b/other/metrics/grafana_seaweedfs.json@@ -1,3500 +1,8599 @@{- "__inputs": [- {- "name": "DS_PROMETHEUS",- "label": "Prometheus",- "description": "",- "type": "datasource",- "pluginId": "prometheus",- "pluginName": "Prometheus"- }- ],- "__requires": [- {- "type": "grafana",- "id": "grafana",- "name": "Grafana",- "version": "8.1.2"- },- {- "type": "panel",- "id": "graph",- "name": "Graph (old)",- "version": ""- },- {- "type": "datasource",- "id": "prometheus",- "name": "Prometheus",- "version": "1.0.0"- }- ],"annotations": {"list": [{"builtIn": 1,- "datasource": "${DS_PROMETHEUS}",+ "datasource": {+ "type": "grafana",+ "uid": "-- Grafana --"+ },"enable": true,"hide": true,"iconColor": "rgba(0, 211, 255, 1)",- "limit": 100,"name": "Annotations & Alerts",- "showIn": 0,"type": "dashboard"}]},"editable": true,- "gnetId": 10423,- "graphTooltip": 0,- "id": null,+ "fiscalYearStartMonth": 0,+ "graphTooltip": 1,"links": [],+ "liveNow": false,"panels": [{- "collapsed": true,- "datasource": null,- "fieldConfig": {- "defaults": {},- "overrides": []- },+ "type": "row",+ "title": "Overview",+ "collapsed": false,+ "id": 101,"gridPos": {"h": 1,"w": 24,"x": 0,"y": 0},- "id": 69,- "panels": [- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "grid": {},- "gridPos": {- "h": 7,- "w": 8,- "x": 0,- "y": 1- },- "id": 46,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null as zero",- "percentage": false,- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le))",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "average",- "refId": "A",- "step": 60- },- {- "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{type}}",- "refId": "B",- "step": 60- }- ],- "thresholds": [],- "timeFrom": null,- "timeShift": null,- "title": "Filer Request Duration 90th percentile",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "s",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": false- }- ]- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "grid": {},- "gridPos": {- "h": 7,- "w": 8,- "x": 8,- "y": 1- },- "id": 49,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null as zero",- "percentage": false,- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le))",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "average",- "refId": "A",- "step": 60- },- {- "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{type}}",- "refId": "B",- "step": 60- },- {- "expr": "",- "format": "time_series",- "intervalFactor": 2,- "refId": "C"- }- ],- "thresholds": [],- "timeFrom": null,- "timeShift": null,- "title": "Filer Request Duration 95th percentile",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "s",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": false- }- ]- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "grid": {},- "gridPos": {- "h": 7,- "w": 8,- "x": 16,- "y": 1- },- "id": 45,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null as zero",- "percentage": false,- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le))",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "average",- "refId": "A",- "step": 60- },- {- "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{type}}",- "refId": "B",- "step": 60- },- {- "expr": "",- "format": "time_series",- "intervalFactor": 2,- "refId": "C"- }- ],- "thresholds": [],- "timeFrom": null,- "timeShift": null,- "title": "Filer Request Duration 99th percentile",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "s",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": false- }- ]- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 0,- "grid": {},- "gridPos": {- "h": 7,- "w": 24,- "x": 0,- "y": 8- },- "id": 2,- "legend": {- "alignAsTable": true,- "avg": false,- "current": true,- "hideEmpty": true,- "hideZero": true,- "max": true,- "min": false,- "rightSide": true,- "show": true,- "sideWidth": 250,- "sort": "max",- "sortDesc": true,- "total": false,- "values": true- },- "lines": true,- "linewidth": 1,- "links": [],- "maxPerRow": 1,- "nullPointMode": "null as zero",- "percentage": false,- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [- {- "alias": "total",- "lines": false- }- ],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "rate(SeaweedFS_filerStore_request_total[1m])",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "{{type}}",- "refId": "A",- "step": 30- }- ],- "thresholds": [],- "timeFrom": null,- "timeShift": null,- "title": "Filer QPS",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 2,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": "0",- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ]- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "gridPos": {- "h": 6,- "w": 12,- "x": 0,- "y": 25- },- "id": 86,- "legend": {- "avg": false,- "current": true,- "max": false,- "min": false,- "show": true,- "total": false,- "values": true- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null",- "percentage": false,- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "SeaweedFS_filer_in_flight_upload_count",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "uploads",- "refId": "A",- "step": 30- }- ],- "thresholds": [],- "timeFrom": null,- "timeShift": null,- "title": "Filer Concurrent Uploads",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 2,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": "0",- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ]- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "gridPos": {- "h": 6,- "w": 12,- "x": 12,- "y": 25- },- "id": 87,- "legend": {- "avg": false,- "current": true,- "max": false,- "min": false,- "show": true,- "total": false,- "values": true- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null",- "percentage": false,- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "SeaweedFS_filer_in_flight_upload_bytes",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "bytes",- "refId": "A",- "step": 30- }- ],- "thresholds": [],- "timeFrom": null,- "timeShift": null,- "title": "Filer Concurrent Upload Bytes",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 2,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "bytes",- "label": null,- "logBase": 1,- "max": null,- "min": "0",- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ]- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 5,- "fillGradient": 0,- "grid": {},- "gridPos": {- "h": 7,- "w": 12,- "x": 0,- "y": 31- },- "hiddenSeries": false,- "id": 91,- "legend": {- "alignAsTable": true,- "avg": true,- "current": true,- "hideEmpty": false,- "hideZero": false,- "max": true,- "min": false,- "rightSide": true,- "show": true,- "total": false,- "values": true- },- "lines": true,- "linewidth": 2,- "links": [],- "nullPointMode": "null as zero",- "options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": true,- "steppedLine": false,- "targets": [- {- "expr": "sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[1m]))",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "< 1KB",- "refId": "A"- },- {- "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[1m])), 0)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "1KB - 100KB",- "refId": "B"- },- {- "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[1m])), 0)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "100KB - 1MB",- "refId": "C"- },- {- "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[1m])), 0)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "1MB - 100MB",- "refId": "D"- },- {- "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[1m])), 0)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "100MB - 1GB",- "refId": "E"- },- {- "expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_count[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[1m])), 0)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "> 1GB",- "refId": "F"- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Filer Object Write Rate by Size Range",- "tooltip": {- "msResolution": false,- "shared": true,- "sort": 0,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "short",- "label": "objects/s",- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }- },- {- "datasource": "${DS_PROMETHEUS}",- "description": "Objects created per size range over the selected time window.",- "fieldConfig": {- "defaults": {- "color": {- "mode": "thresholds"- },- "mappings": [],- "min": 0,- "thresholds": {- "mode": "absolute",- "steps": [- {- "color": "green",- "value": null- }- ]- },- "unit": "short"- },- "overrides": []- },- "gridPos": {- "h": 7,- "w": 12,- "x": 12,- "y": 31- },- "id": 92,- "options": {- "displayMode": "gradient",- "orientation": "horizontal",- "reduceOptions": {- "calcs": [- "lastNotNull"- ],- "fields": "",- "values": false- },- "showUnfilled": true,- "valueMode": "color"- },- "pluginVersion": "8.1.2",- "targets": [- {- "expr": "sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[$__range]))",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "< 1KB",- "refId": "A",- "instant": true- },- {- "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[$__range])), 0)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "1KB - 100KB",- "refId": "B",- "instant": true- },- {- "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[$__range])), 0)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "100KB - 1MB",- "refId": "C",- "instant": true- },- {- "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[$__range])), 0)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "1MB - 100MB",- "refId": "D",- "instant": true- },- {- "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[$__range])), 0)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "100MB - 1GB",- "refId": "E",- "instant": true- },- {- "expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_count[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[$__range])), 0)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "> 1GB",- "refId": "F",- "instant": true- }- ],- "title": "Filer Object Size Distribution",- "type": "bargauge"- }- ],- "repeat": null,- "title": "Filer",- "type": "row"+ "panels": []},{- "collapsed": false,- "datasource": null,- "fieldConfig": {- "defaults": {},- "overrides": []+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"},+ "title": "Masters Up",+ "description": "Master targets currently scrapeable",+ "type": "stat",+ "id": 102,"gridPos": {- "h": 1,- "w": 24,+ "h": 4,+ "w": 3,"x": 0,"y": 1},- "id": 70,- "panels": [- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "gridPos": {- "h": 6,- "w": 12,- "x": 0,- "y": 31+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"},- "id": 88,- "legend": {- "avg": false,- "current": true,- "max": false,- "min": false,- "show": true,- "total": false,- "values": true- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null",- "percentage": false,- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "SeaweedFS_s3_in_flight_upload_count",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "uploads",- "refId": "A",- "step": 30- }- ],- "thresholds": [],- "timeFrom": null,- "timeShift": null,- "title": "S3 Concurrent Uploads",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 2,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": "0",- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ]- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "gridPos": {- "h": 6,- "w": 12,- "x": 12,- "y": 31+ "mappings": [],+ "unit": "none",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "red",+ "value": null+ },+ {+ "color": "green",+ "value": 1+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"},- "id": 89,- "legend": {- "avg": false,- "current": true,- "max": false,- "min": false,- "show": true,- "total": false,- "values": true- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null",- "percentage": false,- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "SeaweedFS_s3_in_flight_upload_bytes",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "bytes",- "refId": "A",- "step": 30- }- ],- "thresholds": [],- "timeFrom": null,- "timeShift": null,- "title": "S3 Concurrent Upload Bytes",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 2,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "bytes",- "label": null,- "logBase": 1,- "max": null,- "min": "0",- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ]+ "editorMode": "code",+ "expr": "sum(up{job=\"seaweedfs-master\", cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A"}],- "repeat": null,- "title": "S3 Gateway",- "type": "row"+ "pluginVersion": "10.3.1"},{- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "fillGradient": 0,- "grid": {},+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Has Leader",+ "description": "",+ "type": "stat",+ "id": 103,"gridPos": {- "h": 7,- "w": 8,- "x": 0,- "y": 2+ "h": 4,+ "w": 3,+ "x": 3,+ "y": 1},- "hiddenSeries": false,- "id": 56,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "bool_yes_no",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "red",+ "value": null+ },+ {+ "color": "green",+ "value": 1+ }+ ]+ }+ },+ "overrides": []},- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null as zero","options": {- "alertThreshold": true+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false},- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,"targets": [{- "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le))",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "average",- "refId": "A",- "step": 60- },- {- "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",- "format": "time_series",- "hide": false,- "interval": "",- "intervalFactor": 2,- "legendFormat": "{{bucket}} {{type}}",- "refId": "B",- "step": 60- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "S3 Request Duration 90th percentile",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "s",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": false+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_master_is_leader{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A"}],- "yaxis": {- "align": false,- "alignLevel": null- }+ "pluginVersion": "10.3.1"},{- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "fillGradient": 0,- "grid": {},+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Volume Servers Up",+ "description": "",+ "type": "stat",+ "id": 104,"gridPos": {- "h": 7,- "w": 8,- "x": 8,- "y": 2+ "h": 4,+ "w": 3,+ "x": 6,+ "y": 1},- "hiddenSeries": false,- "id": 57,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "none",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "red",+ "value": null+ },+ {+ "color": "green",+ "value": 1+ }+ ]+ }+ },+ "overrides": []},- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null as zero","options": {- "alertThreshold": true+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false},- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,"targets": [{- "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le))",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "average",- "refId": "A",- "step": 60- },- {- "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",- "format": "time_series",- "hide": false,- "interval": "",- "intervalFactor": 2,- "legendFormat": "{{bucket}} {{type}}",- "refId": "B",- "step": 60- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "S3 Request Duration 95th percentile",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "s",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": false+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(up{job=\"seaweedfs-volume\", cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A"}],- "yaxis": {- "align": false,- "alignLevel": null- }+ "pluginVersion": "10.3.1"},{- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "fillGradient": 0,- "grid": {},+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Filers Up",+ "description": "",+ "type": "stat",+ "id": 105,"gridPos": {- "h": 7,- "w": 8,- "x": 16,- "y": 2+ "h": 4,+ "w": 3,+ "x": 9,+ "y": 1},- "hiddenSeries": false,- "id": 58,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "none",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "red",+ "value": null+ },+ {+ "color": "green",+ "value": 1+ }+ ]+ }+ },+ "overrides": []},- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null as zero","options": {- "alertThreshold": true+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false},- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,"targets": [{- "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le))",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "average",- "refId": "A",- "step": 60- },- {- "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{bucket}} {{type}}",- "refId": "B",- "step": 60- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "S3 Request Duration 99th percentile",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "s",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": false+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(up{job=\"seaweedfs-filer\", cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A"}],- "yaxis": {- "align": false,- "alignLevel": null- }+ "pluginVersion": "10.3.1"},{- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "fillGradient": 0,+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Volumes",+ "description": "",+ "type": "stat",+ "id": 106,+ "gridPos": {+ "h": 4,+ "w": 3,+ "x": 12,+ "y": 1+ },"fieldConfig": {"defaults": {- "unit": "decbytes"+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }},"overrides": []},- "gridPos": {- "h": 7,- "w": 12,- "x": 0,- "y": 9- },- "hiddenSeries": false,- "id": 84,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null","options": {- "alertThreshold": true+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false},- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,"targets": [{- "expr": "sum(rate(SeaweedFS_s3_bucket_traffic_received_bytes_total[$__rate_interval])) by (bucket)",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{bucket}}",+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_volumeServer_volumes{type=\"volume\", cluster=~\"$cluster\"})",+ "range": true,"refId": "A"}],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "S3 Bucket Traffic Received",- "tooltip": {- "shared": true,- "sort": 0,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "EC Shards",+ "description": "",+ "type": "stat",+ "id": 107,+ "gridPos": {+ "h": 4,+ "w": 3,+ "x": 15,+ "y": 1},- "yaxes": [- {- "format": "decbytes",- "logBase": 1,- "min": 0,- "show": true- },- {- "format": "short",- "logBase": 1,- "show": false- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "fillGradient": 0,"fieldConfig": {"defaults": {- "unit": "decbytes"+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }},"overrides": []},- "gridPos": {- "h": 7,- "w": 12,- "x": 12,- "y": 9- },- "hiddenSeries": false,- "id": 85,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null","options": {- "alertThreshold": true+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false},- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,"targets": [{- "expr": "sum(rate(SeaweedFS_s3_bucket_traffic_sent_bytes_total[$__rate_interval])) by (bucket)",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{bucket}}",+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_volumeServer_volumes{type=\"ec_shards\", cluster=~\"$cluster\"})",+ "range": true,"refId": "A"}],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "S3 Bucket Traffic Sent",- "tooltip": {- "shared": true,- "sort": 0,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "decbytes",- "logBase": 1,- "min": 0,- "show": true- },- {- "format": "short",- "logBase": 1,- "show": false- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }+ "pluginVersion": "10.3.1"},{- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "fillGradient": 0,+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Disk Used",+ "description": "",+ "type": "stat",+ "id": 108,+ "gridPos": {+ "h": 4,+ "w": 3,+ "x": 18,+ "y": 1+ },"fieldConfig": {"defaults": {- "unit": "reqps"+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }},"overrides": []},- "gridPos": {- "h": 7,- "w": 12,- "x": 0,- "y": 16- },- "hiddenSeries": false,- "id": 90,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null","options": {- "alertThreshold": true+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false},- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,"targets": [{- "expr": "sum(rate(SeaweedFS_s3_request_total[$__rate_interval])) by (bucket)",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{bucket}}",+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_volumeServer_total_disk_size{type=\"normal\", cluster=~\"$cluster\"})",+ "range": true,"refId": "A"}],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "S3 API Calls per Bucket",- "tooltip": {- "shared": true,- "sort": 0,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "reqps",- "logBase": 1,- "min": 0,- "show": true- },- {- "format": "short",- "logBase": 1,- "show": false- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }+ "pluginVersion": "10.3.1"},{- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 0,- "fillGradient": 0,- "grid": {},+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Read-only Volumes",+ "description": "",+ "type": "stat",+ "id": 109,"gridPos": {- "h": 7,- "w": 12,- "x": 12,- "y": 16+ "h": 4,+ "w": 3,+ "x": 21,+ "y": 1},- "hiddenSeries": false,- "id": 55,- "legend": {- "alignAsTable": true,- "avg": false,- "current": true,- "hideEmpty": true,- "hideZero": true,- "max": true,- "min": false,- "rightSide": true,- "show": true,- "sideWidth": 250,- "sort": "max",- "sortDesc": true,- "total": false,- "values": true+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ },+ {+ "color": "yellow",+ "value": 1+ }+ ]+ }+ },+ "overrides": []},- "lines": true,- "linewidth": 1,- "links": [],- "maxPerRow": 1,- "nullPointMode": "null as zero","options": {- "alertThreshold": true+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false},- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [- {- "alias": "total",- "lines": false- }- ],- "spaceLength": 10,- "stack": false,- "steppedLine": false,"targets": [{- "expr": "rate(SeaweedFS_s3_request_total[1m])",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "{{bucket}} {{type}}",- "refId": "A",- "step": 30- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "S3 API QPS",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 2,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": "0",- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_volumeServer_read_only_volumes{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A"}],- "yaxis": {- "align": false,- "alignLevel": null- }+ "pluginVersion": "10.3.1"},{- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 0,- "fillGradient": 0,- "grid": {},+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Request Rate by Component",+ "description": "Aggregate request rate per component",+ "type": "timeseries",+ "id": 110,"gridPos": {- "h": 7,- "w": 24,+ "h": 8,+ "w": 12,"x": 0,- "y": 23+ "y": 5},- "hiddenSeries": false,- "hideTimeOverride": false,- "id": 59,- "legend": {- "alignAsTable": true,- "avg": false,- "current": true,- "hideEmpty": true,- "hideZero": true,- "max": true,- "min": false,- "rightSide": true,- "show": true,- "sideWidth": 250,- "sort": "max",- "sortDesc": true,- "total": false,- "values": true+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []},- "lines": true,- "linewidth": 1,- "links": [],- "maxPerRow": 1,- "nullPointMode": "null as zero","options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [- {- "alias": "total",- "lines": false+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"}- ],- "spaceLength": 10,- "stack": false,- "steppedLine": false,+ },"targets": [{- "exemplar": true,- "expr": "sum by (type) (increase(SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST'}[$__range]))*0.000005",- "format": "time_series",- "hide": false,- "interval": "",- "intervalFactor": 2,- "legendFormat": "{{type}} requests",+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(rate(SeaweedFS_filer_request_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,"refId": "A",- "step": 30+ "legendFormat": "filer"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(rate(SeaweedFS_volumeServer_request_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "B",+ "legendFormat": "volume"},{- "exemplar": true,- "expr": "sum(increase(SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST'}[$__range]))*0.000005",- "format": "time_series",- "hide": false,- "interval": "",- "intervalFactor": 2,- "legendFormat": "All PUT, COPY, POST, LIST",+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(rate(SeaweedFS_s3_request_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,"refId": "C",- "step": 30- },- {- "exemplar": true,- "expr": "sum (increase(SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST'}[$__range]))*0.0000004",- "format": "time_series",- "hide": false,- "interval": "",- "intervalFactor": 2,- "legendFormat": "GET and all other",- "refId": "B"- },- {- "exemplar": true,- "expr": "sum by (type) (increase(SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST'}[$__range]))*0.0000004",- "format": "time_series",- "hide": false,- "interval": "",- "intervalFactor": 2,- "legendFormat": "{{type}} requests",- "refId": "D"- }- ],- "thresholds": [],- "timeFrom": "1M",- "timeRegions": [],- "timeShift": null,- "title": "S3 API Monthly Cost if on AWS",- "tooltip": {- "msResolution": true,- "shared": true,- "sort": 2,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "currencyUSD",- "label": "Cost in US$",- "logBase": 1,- "max": null,- "min": "0",- "show": true- },- {- "format": "currencyUSD",- "label": "Write Cost",- "logBase": 1,- "max": null,- "min": null,- "show": false+ "legendFormat": "s3"}],- "yaxis": {- "align": false,- "alignLevel": null- }+ "pluginVersion": "10.3.1"},{- "collapsed": false,- "datasource": null,- "fieldConfig": {- "defaults": {},- "overrides": []- },- "gridPos": {- "h": 1,- "w": 24,- "x": 0,- "y": 30+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"},- "id": 71,- "panels": [],- "repeat": null,- "title": "Volume Server",- "type": "row"- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "fillGradient": 0,- "grid": {},+ "title": "Error Rate",+ "description": "",+ "type": "timeseries",+ "id": 111,"gridPos": {- "h": 7,+ "h": 8,"w": 12,- "x": 0,- "y": 31+ "x": 12,+ "y": 5},- "hiddenSeries": false,- "id": 47,- "legend": {- "alignAsTable": false,- "avg": false,- "current": false,- "max": false,- "min": false,- "show": false,- "total": false,- "values": false+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []},- "lines": true,- "linewidth": 2,- "links": [],- "nullPointMode": "null as zero","options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket[1m])) by (le, exported_instance))",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{exported_instance}}",- "refId": "B"+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true},- {- "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket[1m])) by (le))",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "average",- "refId": "C"+ "tooltip": {+ "mode": "multi",+ "sort": "desc"}- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Volume Server Request Duration 99th percentile",- "tooltip": {- "msResolution": false,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "s",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "fillGradient": 0,- "grid": {},- "gridPos": {- "h": 7,- "w": 12,- "x": 12,- "y": 31- },- "hiddenSeries": false,- "id": 40,- "legend": {- "alignAsTable": true,- "avg": false,- "current": false,- "hideEmpty": true,- "hideZero": true,- "max": false,- "min": false,- "rightSide": true,- "show": true,- "sort": "total",- "sortDesc": true,- "total": true,- "values": true},- "lines": true,- "linewidth": 2,- "links": [],- "nullPointMode": "null as zero",- "options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,"targets": [{- "expr": "sum(rate(SeaweedFS_volumeServer_request_total[1m])) by (type)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "{{type}}",+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(rate(SeaweedFS_upload_error_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,"refId": "A",- "step": 4- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Volume Server QPS",- "tooltip": {- "msResolution": false,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "fillGradient": 0,- "gridPos": {- "h": 7,- "w": 24,- "x": 0,- "y": 38- },- "hiddenSeries": false,- "id": 48,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null",- "options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [+ "legendFormat": "upload errors"+ },{- "expr": "sum(SeaweedFS_volumeServer_volumes) by (collection, type)",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{collection}} {{type}}",- "refId": "A"+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(rate(SeaweedFS_volumeServer_file_read_failures{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "B",+ "legendFormat": "vol read failures"},{- "expr": "sum(SeaweedFS_volumeServer_max_volumes)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "Total",- "refId": "B"- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Volume Count",- "tooltip": {- "shared": true,- "sort": 0,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(rate(SeaweedFS_volumeServer_file_write_failures{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "C",+ "legendFormat": "vol write failures"}],- "yaxis": {- "align": false,- "alignLevel": null- }+ "pluginVersion": "10.3.1"},{- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "fillGradient": 0,+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Build Info",+ "description": "Running build per target (value is constant 1)",+ "type": "timeseries",+ "id": 112,"gridPos": {- "h": 7,+ "h": 6,"w": 24,"x": 0,- "y": 45+ "y": 13},- "hiddenSeries": false,- "id": 50,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "none",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []},- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null","options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "sum(SeaweedFS_volumeServer_total_disk_size) by (collection, type)",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{collection}} {{type}}",- "refId": "A"+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true},- {- "expr": "sum(SeaweedFS_volumeServer_total_disk_size)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "Total",- "refId": "B"+ "tooltip": {+ "mode": "multi",+ "sort": "desc"}- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Used Disk Space by Collection and Type",- "tooltip": {- "shared": true,- "sort": 0,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "bytes",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "fillGradient": 0,- "gridPos": {- "h": 7,- "w": 24,- "x": 0,- "y": 52- },- "hiddenSeries": false,- "id": 51,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false},- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null",- "options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,"targets": [{- "expr": "sum(SeaweedFS_volumeServer_total_disk_size) by (exported_instance)",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{exported_instance}}",- "refId": "A"- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Used Disk Space by Host",- "tooltip": {- "shared": true,- "sort": 0,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "bytes",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "SeaweedFS_build_info{cluster=~\"$cluster\"}",+ "range": true,+ "refId": "A",+ "legendFormat": "{{job}} {{version}} {{goos}}/{{goarch}}"}],- "yaxis": {- "align": false,- "alignLevel": null- }+ "pluginVersion": "10.3.1"},{- "collapsed": false,- "datasource": null,- "fieldConfig": {- "defaults": {},- "overrides": []- },+ "type": "row",+ "title": "Master",+ "collapsed": true,+ "id": 192,"gridPos": {"h": 1,"w": 24,"x": 0,- "y": 59- },- "id": 72,- "panels": [],- "repeat": null,- "title": "Filer Store",- "type": "row"- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "fillGradient": 0,- "grid": {},- "gridPos": {- "h": 7,- "w": 12,- "x": 0,- "y": 60- },- "hiddenSeries": false,- "id": 12,- "legend": {- "alignAsTable": false,- "avg": false,- "current": false,- "max": false,- "min": false,- "show": false,- "total": false,- "values": false- },- "lines": true,- "linewidth": 2,- "links": [],- "nullPointMode": "null as zero",- "options": {- "alertThreshold": true+ "y": 19},- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [+ "panels": [{- "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[1m])) by (le, type))",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "{{type}}",- "refId": "B"- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Filer Store Request Duration 99th percentile",- "tooltip": {- "msResolution": false,- "shared": true,- "sort": 0,- "value_type": "cumulative"+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Raft Leader",+ "description": "",+ "type": "stat",+ "id": 113,+ "gridPos": {+ "h": 5,+ "w": 6,+ "x": 0,+ "y": 21+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "bool_yes_no",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "red",+ "value": null+ },+ {+ "color": "green",+ "value": 1+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "SeaweedFS_master_is_leader{cluster=~\"$cluster\"}",+ "range": true,+ "refId": "A",+ "legendFormat": "{{instance}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Leader Changes",+ "description": "Raft leadership changes",+ "type": "timeseries",+ "id": 114,+ "gridPos": {+ "h": 8,+ "w": 9,+ "x": 6,+ "y": 21+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "none",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (instance) (increase(SeaweedFS_master_leader_changes{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{instance}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Received Heartbeats",+ "description": "",+ "type": "timeseries",+ "id": 115,+ "gridPos": {+ "h": 8,+ "w": 9,+ "x": 15,+ "y": 21+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (rate(SeaweedFS_master_received_heartbeats{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Volume Layout Writable",+ "description": "",+ "type": "timeseries",+ "id": 116,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 0,+ "y": 29+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (collection) (SeaweedFS_master_volume_layout_writable{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{collection}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Volume Layout Crowded",+ "description": "",+ "type": "timeseries",+ "id": 117,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 8,+ "y": 29+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (collection) (SeaweedFS_master_volume_layout_crowded{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{collection}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Replica Placement Mismatch",+ "description": "Volumes whose replica placement does not match policy",+ "type": "timeseries",+ "id": 118,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 16,+ "y": 29+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (collection) (SeaweedFS_master_replica_placement_mismatch{cluster=~\"$cluster\"} > 0)",+ "range": true,+ "refId": "A",+ "legendFormat": "{{collection}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Pick-for-write / broadcast errors",+ "description": "",+ "type": "timeseries",+ "id": 119,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 37+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "rate(SeaweedFS_master_pick_for_write_error{cluster=~\"$cluster\"}[$__rate_interval])",+ "range": true,+ "refId": "A",+ "legendFormat": "pick_for_write"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "rate(SeaweedFS_master_broadcast_to_full{cluster=~\"$cluster\"}[$__rate_interval])",+ "range": true,+ "refId": "B",+ "legendFormat": "broadcast_to_full"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Master Client Updates",+ "description": "masterclient leader-connection updates by type",+ "type": "timeseries",+ "id": 120,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 37+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (rate(SeaweedFS_wdclient_connect_updates{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Master Uptime",+ "description": "time since master process start",+ "type": "timeseries",+ "id": 121,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 45+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "time() - SeaweedFS_master_start_time_seconds{cluster=~\"$cluster\"}",+ "range": true,+ "refId": "A",+ "legendFormat": "{{instance}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Admin Lock",+ "description": "",+ "type": "stat",+ "id": 122,+ "gridPos": {+ "h": 4,+ "w": 12,+ "x": 12,+ "y": 45+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "SeaweedFS_master_admin_lock{cluster=~\"$cluster\"}",+ "range": true,+ "refId": "A",+ "legendFormat": "{{client}}"+ }+ ],+ "pluginVersion": "10.3.1"+ }+ ]+ },+ {+ "type": "row",+ "title": "Volume Servers",+ "collapsed": true,+ "id": 193,+ "gridPos": {+ "h": 1,+ "w": 24,+ "x": 0,+ "y": 20+ },+ "panels": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Volume Server QPS",+ "description": "",+ "type": "timeseries",+ "id": 123,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 22+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (rate(SeaweedFS_volumeServer_request_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Request Duration p99 by type",+ "description": "",+ "type": "timeseries",+ "id": 124,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 22+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, type))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "In-flight Requests",+ "description": "",+ "type": "timeseries",+ "id": 125,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 30+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (SeaweedFS_volumeServer_in_flight_requests{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Volume / Shard Count",+ "description": "",+ "type": "timeseries",+ "id": 126,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 30+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (collection, type) (SeaweedFS_volumeServer_volumes{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{collection}}/{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Read-only Volumes",+ "description": "",+ "type": "timeseries",+ "id": 127,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 0,+ "y": 38+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (collection) (SeaweedFS_volumeServer_read_only_volumes{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{collection}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Disk Used by Collection/Type",+ "description": "",+ "type": "timeseries",+ "id": 128,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 8,+ "y": 38+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (collection, type) (SeaweedFS_volumeServer_total_disk_size{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{collection}}/{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Disk Used by Host",+ "description": "",+ "type": "timeseries",+ "id": 129,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 16,+ "y": 38+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (instance) (SeaweedFS_volumeServer_total_disk_size{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{instance}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Disk Resource (per location)",+ "description": "Disk space per storage location",+ "type": "timeseries",+ "id": 130,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 46+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "SeaweedFS_volumeServer_resource{type=\"Used\", cluster=~\"$cluster\"}",+ "range": true,+ "refId": "A",+ "legendFormat": "used {{instance}} {{name}}"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "SeaweedFS_volumeServer_resource{type=\"All\", cluster=~\"$cluster\"}",+ "range": true,+ "refId": "B",+ "legendFormat": "all {{instance}} {{name}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Disk Errors",+ "description": "Disk locations reporting an error (0 = healthy)",+ "type": "stat",+ "id": 131,+ "gridPos": {+ "h": 4,+ "w": 4,+ "x": 12,+ "y": 46+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ },+ {+ "color": "red",+ "value": 1+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_volumeServer_disk_error_status{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Max Volumes",+ "description": "",+ "type": "stat",+ "id": 132,+ "gridPos": {+ "h": 4,+ "w": 4,+ "x": 16,+ "y": 46+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_volumeServer_max_volumes{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Vacuum Operations",+ "description": "",+ "type": "timeseries",+ "id": 133,+ "gridPos": {+ "h": 8,+ "w": 16,+ "x": 0,+ "y": 54+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ops",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (success) (rate(SeaweedFS_volumeServer_vacuuming_compact_count{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "compact success={{success}}"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (success) (rate(SeaweedFS_volumeServer_vacuuming_commit_count{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "B",+ "legendFormat": "commit success={{success}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Vacuuming Duration p99",+ "description": "",+ "type": "timeseries",+ "id": 134,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 62+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_vacuuming_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, type))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "File Read/Write Failures",+ "description": "",+ "type": "timeseries",+ "id": 135,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 62+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "rate(SeaweedFS_volumeServer_file_read_failures{cluster=~\"$cluster\"}[$__rate_interval])",+ "range": true,+ "refId": "A",+ "legendFormat": "read {{instance}}"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "rate(SeaweedFS_volumeServer_file_write_failures{cluster=~\"$cluster\"}[$__rate_interval])",+ "range": true,+ "refId": "B",+ "legendFormat": "write {{instance}}"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "rate(SeaweedFS_volumeServer_file_read_invalid_needles{cluster=~\"$cluster\"}[$__rate_interval])",+ "range": true,+ "refId": "C",+ "legendFormat": "invalid needle {{instance}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Master Disconnections",+ "description": "",+ "type": "timeseries",+ "id": 136,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 70+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ops",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (instance) (rate(SeaweedFS_volumeServer_master_disconnections{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{instance}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "In-flight & Concurrent Limits",+ "description": "",+ "type": "timeseries",+ "id": 137,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 70+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_volumeServer_in_flight_upload_size{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "upload in-flight"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_volumeServer_in_flight_download_size{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "B",+ "legendFormat": "download in-flight"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_volumeServer_concurrent_upload_limit{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "C",+ "legendFormat": "upload limit"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_volumeServer_concurrent_download_limit{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "D",+ "legendFormat": "download limit"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Handler Calls",+ "description": "",+ "type": "timeseries",+ "id": 138,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 78+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ops",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (rate(SeaweedFS_volumeServer_handler_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Scrub Failures",+ "description": "",+ "type": "timeseries",+ "id": 139,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 78+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ops",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (mode) (rate(SeaweedFS_volumeServer_scrub_volume_failures{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "volume {{mode}}"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (mode) (rate(SeaweedFS_volumeServer_scrub_shard_failures{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "B",+ "legendFormat": "shard {{mode}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Time Since Last Scrub",+ "description": "",+ "type": "timeseries",+ "id": 140,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 86+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "time() - max by (mode) (SeaweedFS_volumeServer_scrub_last_time_seconds{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{mode}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Volume Server Uptime",+ "description": "",+ "type": "timeseries",+ "id": 141,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 86+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "time() - SeaweedFS_volumeServer_start_time_seconds{cluster=~\"$cluster\"}",+ "range": true,+ "refId": "A",+ "legendFormat": "{{instance}}"+ }+ ],+ "pluginVersion": "10.3.1"+ }+ ]+ },+ {+ "type": "row",+ "title": "Filer",+ "collapsed": true,+ "id": 194,+ "gridPos": {+ "h": 1,+ "w": 24,+ "x": 0,+ "y": 21+ },+ "panels": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Filer QPS",+ "description": "",+ "type": "timeseries",+ "id": 142,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 23+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (rate(SeaweedFS_filer_request_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Filer Request Errors by code",+ "description": "",+ "type": "timeseries",+ "id": 143,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 23+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (code) (rate(SeaweedFS_filer_request_total{code!~\"2..\", cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{code}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Filer Request Duration p99",+ "description": "",+ "type": "timeseries",+ "id": 144,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 0,+ "y": 31+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, type))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Filer Request Duration p95",+ "description": "",+ "type": "timeseries",+ "id": 145,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 8,+ "y": 31+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, type))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Filer Request Duration p50",+ "description": "",+ "type": "timeseries",+ "id": 146,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 16,+ "y": 31+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.50, sum(rate(SeaweedFS_filer_request_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, type))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "In-flight Requests",+ "description": "",+ "type": "timeseries",+ "id": 147,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 0,+ "y": 39+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (SeaweedFS_filer_in_flight_requests{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "In-flight Uploads",+ "description": "",+ "type": "timeseries",+ "id": 148,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 8,+ "y": 39+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_filer_in_flight_upload_count{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "count"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "In-flight Upload Bytes",+ "description": "",+ "type": "timeseries",+ "id": 149,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 16,+ "y": 39+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_filer_in_flight_upload_bytes{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "bytes"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Object Size Distribution",+ "description": "Size of newly-created objects",+ "type": "timeseries",+ "id": 150,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 47+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_object_size_bytes_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le))",+ "range": true,+ "refId": "A",+ "legendFormat": "p99"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.50, sum(rate(SeaweedFS_filer_object_size_bytes_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le))",+ "range": true,+ "refId": "B",+ "legendFormat": "p50"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Handler Calls",+ "description": "",+ "type": "timeseries",+ "id": 151,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 47+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ops",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (rate(SeaweedFS_filer_handler_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Upload Errors",+ "description": "",+ "type": "timeseries",+ "id": 152,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 55+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (code) (rate(SeaweedFS_upload_error_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{code}}"+ }+ ],+ "pluginVersion": "10.3.1"+ }+ ]+ },+ {+ "type": "row",+ "title": "Filer Store / Sync",+ "collapsed": true,+ "id": 195,+ "gridPos": {+ "h": 1,+ "w": 24,+ "x": 0,+ "y": 22+ },+ "panels": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Filer Store QPS",+ "description": "",+ "type": "timeseries",+ "id": 153,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 24+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (store, type) (rate(SeaweedFS_filerStore_request_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{store}}/{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Filer Store Duration p99",+ "description": "",+ "type": "timeseries",+ "id": 154,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 24+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, type))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Filer Sync Offset Lag",+ "description": "Seconds behind source (filer.sync); only present when filer sync runs",+ "type": "timeseries",+ "id": 155,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 32+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "time() - SeaweedFS_filerSync_sync_offset{cluster=~\"$cluster\"} / 1e9",+ "range": true,+ "refId": "A",+ "legendFormat": "{{clientName}} {{path}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Metadata Subscription Lag",+ "description": "Seconds since the filer last sent a metadata-subscription event (per client/path); only present with active subscribers",+ "type": "timeseries",+ "id": 156,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 32+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "time() - max by (clientName, path) (SeaweedFS_filer_last_send_timestamp_of_subscribe{cluster=~\"$cluster\"}) / 1e9",+ "range": true,+ "refId": "A",+ "legendFormat": "{{clientName}} {{path}}"+ }+ ],+ "pluginVersion": "10.3.1"+ }+ ]+ },+ {+ "type": "row",+ "title": "S3 Gateway",+ "collapsed": true,+ "id": 196,+ "gridPos": {+ "h": 1,+ "w": 24,+ "x": 0,+ "y": 23},- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []+ "panels": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "S3 QPS by type",+ "description": "",+ "type": "timeseries",+ "id": 157,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 25+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (rate(SeaweedFS_s3_request_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "S3 Errors by code",+ "description": "",+ "type": "timeseries",+ "id": 158,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 25+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "reqps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (code) (rate(SeaweedFS_s3_request_total{code!~\"2..\", cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{code}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "S3 Request Duration p99",+ "description": "",+ "type": "timeseries",+ "id": 159,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 33+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, type))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "S3 Time-to-First-Byte p99",+ "description": "",+ "type": "timeseries",+ "id": 160,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 33+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ms",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_time_to_first_byte_millisecond_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, type))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "S3 In-flight Requests",+ "description": "",+ "type": "timeseries",+ "id": 161,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 0,+ "y": 41+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (SeaweedFS_s3_in_flight_requests{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "S3 In-flight Uploads",+ "description": "in-flight upload count and bytes",+ "type": "timeseries",+ "id": 162,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 8,+ "y": 41+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_s3_in_flight_upload_count{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "count"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_s3_in_flight_upload_bytes{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "B",+ "legendFormat": "bytes"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "S3 Handler Events",+ "description": "",+ "type": "timeseries",+ "id": 163,+ "gridPos": {+ "h": 8,+ "w": 8,+ "x": 16,+ "y": 41+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ops",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (rate(SeaweedFS_s3_handler_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ }+ ]+ },+ {+ "type": "row",+ "title": "S3 Buckets",+ "collapsed": true,+ "id": 197,+ "gridPos": {+ "h": 1,+ "w": 24,+ "x": 0,+ "y": 24},- "yaxes": [- {- "format": "s",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true+ "panels": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Bucket Logical Size",+ "description": "",+ "type": "timeseries",+ "id": 164,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 26+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "max by (bucket) (SeaweedFS_s3_bucket_size_bytes{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{bucket}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Bucket Physical Size",+ "description": "",+ "type": "timeseries",+ "id": 165,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 26+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "max by (bucket) (SeaweedFS_s3_bucket_physical_size_bytes{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{bucket}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Bucket Object Count",+ "description": "",+ "type": "timeseries",+ "id": 166,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 34+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "max by (bucket) (SeaweedFS_s3_bucket_object_count{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{bucket}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Bucket Traffic",+ "description": "",+ "type": "timeseries",+ "id": 167,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 34+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "Bps",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (bucket) (rate(SeaweedFS_s3_bucket_traffic_received_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "in {{bucket}}"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (bucket) (rate(SeaweedFS_s3_bucket_traffic_sent_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "B",+ "legendFormat": "out {{bucket}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Objects Uploaded / Deleted",+ "description": "",+ "type": "timeseries",+ "id": 168,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 42+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ops",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (bucket) (rate(SeaweedFS_s3_uploaded_objects{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "uploaded {{bucket}}"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (bucket) (rate(SeaweedFS_s3_deleted_objects{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "B",+ "legendFormat": "deleted {{bucket}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Bucket Quota & Read-only",+ "description": "Quota bytes (and read-only flag) per bucket",+ "type": "timeseries",+ "id": 169,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 42+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "max by (bucket) (SeaweedFS_s3_bucket_quota_bytes{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "quota {{bucket}}"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "max by (bucket) (SeaweedFS_s3_bucket_read_only{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "B",+ "legendFormat": "read-only {{bucket}}"+ }+ ],+ "pluginVersion": "10.3.1"}- ],- "yaxis": {- "align": false,- "alignLevel": null- }+ ]},{- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "fillGradient": 0,- "grid": {},+ "type": "row",+ "title": "S3 Lifecycle",+ "collapsed": true,+ "id": 198,"gridPos": {- "h": 7,- "w": 12,- "x": 12,- "y": 60- },- "hiddenSeries": false,- "id": 14,- "legend": {- "alignAsTable": true,- "avg": true,- "current": true,- "hideEmpty": false,- "hideZero": false,- "max": false,- "min": false,- "rightSide": true,- "show": true,- "total": false,- "values": true- },- "lines": true,- "linewidth": 2,- "links": [],- "nullPointMode": "null as zero",- "options": {- "alertThreshold": true+ "h": 1,+ "w": 24,+ "x": 0,+ "y": 25},- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [+ "panels": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Lifecycle Dispatch Outcomes",+ "description": "",+ "type": "timeseries",+ "id": 170,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 27+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ops",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (kind, outcome) (rate(SeaweedFS_s3_lifecycle_dispatch_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{kind}}/{{outcome}}"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (kind) (rate(SeaweedFS_s3_lifecycle_bootstrap_dispatch_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "B",+ "legendFormat": "bootstrap {{kind}}"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum (rate(SeaweedFS_s3_lifecycle_metadata_only_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "C",+ "legendFormat": "metadata-only (TTL reclaim)"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Lifecycle Schedule Depth",+ "description": "",+ "type": "timeseries",+ "id": 171,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 27+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (shard) (SeaweedFS_s3_lifecycle_schedule_depth{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "shard {{shard}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Lifecycle Cursor Lag",+ "description": "now - min cursor timestamp",+ "type": "timeseries",+ "id": 172,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 35+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "time() - max by (shard) (SeaweedFS_s3_lifecycle_cursor_min_ts_ns{cluster=~\"$cluster\"}) / 1e9",+ "range": true,+ "refId": "A",+ "legendFormat": "shard {{shard}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Lifecycle Events Scanned",+ "description": "",+ "type": "timeseries",+ "id": 173,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 35+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ops",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (shard) (rate(SeaweedFS_s3_lifecycle_events_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "router {{shard}}"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (shard) (rate(SeaweedFS_s3_lifecycle_daily_run_events_scanned_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "B",+ "legendFormat": "daily-replay {{shard}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Daily Run Duration p99",+ "description": "",+ "type": "timeseries",+ "id": 174,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 43+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_lifecycle_daily_run_shard_duration_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, shard))",+ "range": true,+ "refId": "A",+ "legendFormat": "shard {{shard}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Dispatch Limiter Wait p99",+ "description": "",+ "type": "timeseries",+ "id": 175,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 43+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_lifecycle_dispatch_limiter_wait_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le))",+ "range": true,+ "refId": "A",+ "legendFormat": "p99"+ }+ ],+ "pluginVersion": "10.3.1"+ },{- "expr": "sum(rate(SeaweedFS_filerStore_request_total [1m])) by (type)",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "{{type}}",- "refId": "B"- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Filer Store QPS",- "tooltip": {- "msResolution": false,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Time Since Last Daily Walk",+ "description": "0/never excluded; lower is fresher",+ "type": "timeseries",+ "id": 176,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 51+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "time() - max by (shard) (SeaweedFS_s3_lifecycle_daily_run_last_walked_ns{cluster=~\"$cluster\"}) / 1e9",+ "range": true,+ "refId": "A",+ "legendFormat": "shard {{shard}}"+ }+ ],+ "pluginVersion": "10.3.1"}- ],- "yaxis": {- "align": false,- "alignLevel": null- }+ ]},{- "collapsed": false,- "datasource": null,- "fieldConfig": {- "defaults": {},- "overrides": []- },+ "type": "row",+ "title": "Admin / Maintenance",+ "collapsed": true,+ "id": 199,"gridPos": {"h": 1,"w": 24,"x": 0,- "y": 67- },- "id": 73,- "panels": [],- "repeat": null,- "title": "Filer Instances",- "type": "row"- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "fillGradient": 0,- "grid": {},- "gridPos": {- "h": 7,- "w": 12,- "x": 0,- "y": 68- },- "hiddenSeries": false,- "id": 52,- "legend": {- "alignAsTable": false,- "avg": false,- "current": false,- "max": false,- "min": false,- "show": false,- "total": false,- "values": false+ "y": 26},- "lines": true,- "linewidth": 2,- "links": [],- "nullPointMode": "null as zero",- "options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [+ "panels": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Maintenance Tasks by Status",+ "description": "",+ "type": "timeseries",+ "id": 177,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 28+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (status) (SeaweedFS_admin_maintenance_tasks_by_status{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{status}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Maintenance Tasks by Type",+ "description": "",+ "type": "timeseries",+ "id": 178,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 28+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type) (SeaweedFS_admin_maintenance_tasks_by_type{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Tasks Completed",+ "description": "",+ "type": "timeseries",+ "id": 179,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 36+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ops",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (type, outcome) (rate(SeaweedFS_admin_maintenance_tasks_completed_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}/{{outcome}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Task Duration p99",+ "description": "",+ "type": "timeseries",+ "id": 180,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 36+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_admin_maintenance_task_duration_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le, type))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{type}}"+ }+ ],+ "pluginVersion": "10.3.1"+ },{- "expr": "go_memstats_alloc_bytes{exported_job=\"filer\"}",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "bytes allocated",- "refId": "B"+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Workers Connected",+ "description": "",+ "type": "stat",+ "id": 181,+ "gridPos": {+ "h": 4,+ "w": 6,+ "x": 0,+ "y": 44+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum(SeaweedFS_admin_workers_connected{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A"+ }+ ],+ "pluginVersion": "10.3.1"},{- "expr": "rate(go_memstats_alloc_bytes_total{exported_job=\"filer\"}[30s])",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "alloc rate",- "refId": "A"+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Worker Slots",+ "description": "",+ "type": "timeseries",+ "id": 182,+ "gridPos": {+ "h": 8,+ "w": 9,+ "x": 6,+ "y": 44+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (state) (SeaweedFS_admin_worker_slots{cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{state}}"+ }+ ],+ "pluginVersion": "10.3.1"},{- "expr": "go_memstats_stack_inuse_bytes{exported_job=\"filer\"}",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "stack inuse",- "refId": "C"+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Worker Events",+ "description": "",+ "type": "timeseries",+ "id": 183,+ "gridPos": {+ "h": 8,+ "w": 9,+ "x": 15,+ "y": 44+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "ops",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (event) (rate(SeaweedFS_admin_worker_events_total{cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{event}}"+ }+ ],+ "pluginVersion": "10.3.1"},{- "expr": "go_memstats_heap_inuse_bytes{exported_job=\"filer\"}",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "heap inuse",- "refId": "D"- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Filer Go Memory Stats",- "tooltip": {- "msResolution": false,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "bytes",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "Bps",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "fillGradient": 0,- "grid": {},- "gridPos": {- "h": 7,- "w": 12,- "x": 12,- "y": 68- },- "hiddenSeries": false,- "id": 54,- "legend": {- "alignAsTable": false,- "avg": false,- "current": false,- "max": false,- "min": false,- "show": false,- "total": false,- "values": false- },- "lines": true,- "linewidth": 2,- "links": [],- "nullPointMode": "null as zero",- "options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Last Maintenance Scan",+ "description": "",+ "type": "stat",+ "id": 184,+ "gridPos": {+ "h": 4,+ "w": 6,+ "x": 0,+ "y": 52+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "dateTimeFromNow",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "max(SeaweedFS_admin_maintenance_last_scan_timestamp_seconds{cluster=~\"$cluster\"}) * 1000",+ "range": true,+ "refId": "A"+ }+ ],+ "pluginVersion": "10.3.1"+ },{- "expr": "go_gc_duration_seconds{exported_job=\"filer\"}",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "{{quantile}}",- "refId": "B"- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Filer Go GC duration quantiles",- "tooltip": {- "msResolution": false,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "s",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "Bps",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Next Maintenance Scan",+ "description": "",+ "type": "stat",+ "id": 185,+ "gridPos": {+ "h": 4,+ "w": 6,+ "x": 6,+ "y": 52+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "thresholds"+ },+ "mappings": [],+ "unit": "dateTimeFromNow",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "colorMode": "value",+ "graphMode": "area",+ "justifyMode": "auto",+ "orientation": "auto",+ "reduceOptions": {+ "calcs": [+ "lastNotNull"+ ],+ "fields": "",+ "values": false+ },+ "textMode": "auto",+ "wideLayout": true,+ "showPercentChange": false+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "max(SeaweedFS_admin_maintenance_next_scan_timestamp_seconds{cluster=~\"$cluster\"}) * 1000",+ "range": true,+ "refId": "A"+ }+ ],+ "pluginVersion": "10.3.1"}- ],- "yaxis": {- "align": false,- "alignLevel": null- }+ ]},{- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fill": 1,- "fillGradient": 0,- "grid": {},+ "type": "row",+ "title": "Go Runtime",+ "collapsed": true,+ "id": 200,"gridPos": {- "h": 7,+ "h": 1,"w": 24,"x": 0,- "y": 75- },- "hiddenSeries": false,- "id": 53,- "legend": {- "alignAsTable": false,- "avg": false,- "current": false,- "max": false,- "min": false,- "show": false,- "total": false,- "values": false+ "y": 27},- "lines": true,- "linewidth": 2,- "links": [],- "nullPointMode": "null as zero",- "options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [+ "panels": [{- "expr": "go_goroutines{exported_job=\"filer\"}",- "format": "time_series",- "intervalFactor": 2,- "legendFormat": "{{exported_instance}}",- "refId": "B"- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Filer Go Routines",- "tooltip": {- "msResolution": false,- "shared": true,- "sort": 0,- "value_type": "cumulative"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "none",- "label": null,- "logBase": 1,- "max": null,- "min": 0,- "show": true- },- {- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fieldConfig": {- "defaults": {- "links": []+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Goroutines by component",+ "description": "",+ "type": "timeseries",+ "id": 186,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 29+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (job) (go_goroutines{job=~\"seaweedfs-.*\", cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{job}}"+ }+ ],+ "pluginVersion": "10.3.1"},- "overrides": []- },- "fill": 1,- "fillGradient": 0,- "gridPos": {- "h": 7,- "w": 8,- "x": 0,- "y": 82- },- "hiddenSeries": false,- "id": 66,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null",- "options": {- "alertThreshold": true- },- "paceLength": 10,- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [- {- "$$hashKey": "object:258",- "alias": "max limit",- "color": "#890f02"- }- ],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [{- "expr": "process_resident_memory_bytes{exported_job=\"filer\"}",- "format": "time_series",- "groupBy": [- {- "params": [- "$__interval"- ],- "type": "time"+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Heap In-use by component",+ "description": "",+ "type": "timeseries",+ "id": 187,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 29+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }},- {- "params": [- "null"+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"],- "type": "fill"+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"}- ],- "interval": "",- "intervalFactor": 2,- "legendFormat": "resident {{exported_instance}}",- "orderByTime": "ASC",- "policy": "default",- "refId": "A",- "resultFormat": "time_series",- "select": [- [- {- "params": [- "value"- ],- "type": "field"+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"},- {- "params": [],- "type": "mean"- }- ]+ "editorMode": "code",+ "expr": "sum by (job) (go_memstats_heap_inuse_bytes{job=~\"seaweedfs-.*\", cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{job}}"+ }],- "step": 120,- "tags": []- },- {- "expr": "process_virtual_memory_bytes{exported_job=\"filer\"}",- "format": "time_series",- "interval": "",- "intervalFactor": 2,- "legendFormat": "virtual {{exported_instance}}",- "refId": "B",- "step": 120- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Process memory",- "tooltip": {- "shared": true,- "sort": 2,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "$$hashKey": "object:265",- "format": "bytes",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- },- {- "$$hashKey": "object:266",- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "editable": true,- "error": false,- "fieldConfig": {- "defaults": {- "links": []+ "pluginVersion": "10.3.1"},- "overrides": []- },- "fill": 1,- "fillGradient": 0,- "gridPos": {- "h": 7,- "w": 8,- "x": 8,- "y": 82- },- "hiddenSeries": false,- "id": 68,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "connected",- "options": {- "alertThreshold": true- },- "paceLength": 10,- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [{- "expr": "process_open_fds{exported_job=\"filer\"}",- "format": "time_series",- "groupBy": [- {- "params": [- "$interval"+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "GC Pause p99 by component",+ "description": "",+ "type": "timeseries",+ "id": 188,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 37+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "s",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"],- "type": "time"+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true},+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [{- "params": [- "null"- ],- "type": "fill"+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "max by (job) (go_gc_duration_seconds{quantile=\"1\", job=~\"seaweedfs-.*\", cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{job}}"}],- "interval": "",- "intervalFactor": 2,- "legendFormat": "{{exported_instance}}",- "metric": "",- "policy": "default",- "refId": "A",- "resultFormat": "time_series",- "select": [- [- {- "params": [- "value"- ],- "type": "field"+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Process CPU by component",+ "description": "",+ "type": "timeseries",+ "id": 189,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 37+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"},- {- "params": [],- "type": "mean"+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "percentunit",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]}- ]+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (job) (rate(process_cpu_seconds_total{job=~\"seaweedfs-.*\", cluster=~\"$cluster\"}[$__rate_interval]))",+ "range": true,+ "refId": "A",+ "legendFormat": "{{job}}"+ }],- "step": 240,- "tags": []+ "pluginVersion": "10.3.1"},{- "expr": "process_max_fds{exported_job=\"filer\"}",- "format": "time_series",- "groupBy": [- {- "params": [- "$interval"+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Process RSS by component",+ "description": "",+ "type": "timeseries",+ "id": 190,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 0,+ "y": 45+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"+ },+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "bytes",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]+ }+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"],- "type": "time"+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true},+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [{- "params": [- "null"- ],- "type": "fill"+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (job) (process_resident_memory_bytes{job=~\"seaweedfs-.*\", cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{job}}"}],- "hide": false,- "interval": "",- "intervalFactor": 2,- "legendFormat": "{{exported_instance}}",- "metric": "",- "policy": "default",- "refId": "B",- "resultFormat": "time_series",- "select": [- [- {- "params": [- "value"- ],- "type": "field"+ "pluginVersion": "10.3.1"+ },+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "title": "Open File Descriptors",+ "description": "",+ "type": "timeseries",+ "id": 191,+ "gridPos": {+ "h": 8,+ "w": 12,+ "x": 12,+ "y": 45+ },+ "fieldConfig": {+ "defaults": {+ "color": {+ "mode": "palette-classic"},- {- "params": [],- "type": "mean"+ "custom": {+ "axisBorderShow": false,+ "axisCenteredZero": false,+ "axisColorMode": "text",+ "axisLabel": "",+ "axisPlacement": "auto",+ "barAlignment": 0,+ "drawStyle": "line",+ "fillOpacity": 10,+ "gradientMode": "none",+ "hideFrom": {+ "legend": false,+ "tooltip": false,+ "viz": false+ },+ "insertNulls": false,+ "lineInterpolation": "linear",+ "lineWidth": 1,+ "pointSize": 4,+ "scaleDistribution": {+ "type": "linear"+ },+ "showPoints": "never",+ "spanNulls": false,+ "stacking": {+ "group": "A",+ "mode": "none"+ },+ "thresholdsStyle": {+ "mode": "off"+ }+ },+ "mappings": [],+ "unit": "short",+ "thresholds": {+ "mode": "absolute",+ "steps": [+ {+ "color": "green",+ "value": null+ }+ ]}- ]+ },+ "overrides": []+ },+ "options": {+ "legend": {+ "calcs": [+ "lastNotNull",+ "max"+ ],+ "displayMode": "table",+ "placement": "bottom",+ "showLegend": true+ },+ "tooltip": {+ "mode": "multi",+ "sort": "desc"+ }+ },+ "targets": [+ {+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "editorMode": "code",+ "expr": "sum by (job) (process_open_fds{job=~\"seaweedfs-.*\", cluster=~\"$cluster\"})",+ "range": true,+ "refId": "A",+ "legendFormat": "{{job}}"+ }],- "step": 240,- "tags": []- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "Open File Descriptor",- "tooltip": {- "msResolution": false,- "shared": true,- "sort": 0,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "$$hashKey": "object:1087",- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- },- {- "$$hashKey": "object:1088",- "format": "short",- "label": null,- "logBase": 1,- "max": null,- "min": null,- "show": true- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "fillGradient": 0,- "fieldConfig": {- "defaults": {- "unit": "decbytes"- },- "overrides": []- },- "gridPos": {- "h": 7,- "w": 12,- "x": 0,- "y": 37- },- "hiddenSeries": false,- "id": 87,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null as zero",- "options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "max(SeaweedFS_s3_bucket_size_bytes) by (bucket)",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{bucket}} (logical)",- "refId": "A"- },- {- "expr": "max(SeaweedFS_s3_bucket_physical_size_bytes) by (bucket)",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{bucket}} (physical)",- "refId": "B"- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "S3 Bucket Size",- "tooltip": {- "shared": true,- "sort": 0,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "decbytes",- "logBase": 1,- "min": 0,- "show": true- },- {- "format": "short",- "logBase": 1,- "show": false- }- ],- "yaxis": {- "align": false,- "alignLevel": null- }- },- {- "aliasColors": {},- "bars": false,- "dashLength": 10,- "dashes": false,- "datasource": "${DS_PROMETHEUS}",- "fill": 1,- "fillGradient": 0,- "fieldConfig": {- "defaults": {- "unit": "short"- },- "overrides": []- },- "gridPos": {- "h": 7,- "w": 12,- "x": 12,- "y": 37- },- "hiddenSeries": false,- "id": 88,- "legend": {- "avg": false,- "current": false,- "max": false,- "min": false,- "show": true,- "total": false,- "values": false- },- "lines": true,- "linewidth": 1,- "links": [],- "nullPointMode": "null as zero",- "options": {- "alertThreshold": true- },- "percentage": false,- "pluginVersion": "8.1.2",- "pointradius": 5,- "points": false,- "renderer": "flot",- "seriesOverrides": [],- "spaceLength": 10,- "stack": false,- "steppedLine": false,- "targets": [- {- "expr": "max(SeaweedFS_s3_bucket_object_count) by (bucket)",- "format": "time_series",- "hide": false,- "intervalFactor": 2,- "legendFormat": "{{bucket}}",- "refId": "A"- }- ],- "thresholds": [],- "timeFrom": null,- "timeRegions": [],- "timeShift": null,- "title": "S3 Bucket Object Count",- "tooltip": {- "shared": true,- "sort": 0,- "value_type": "individual"- },- "type": "graph",- "xaxis": {- "buckets": null,- "mode": "time",- "name": null,- "show": true,- "values": []- },- "yaxes": [- {- "format": "short",- "logBase": 1,- "min": 0,- "show": true- },- {- "format": "short",- "logBase": 1,- "show": false+ "pluginVersion": "10.3.1"}- ],- "yaxis": {- "align": false,- "alignLevel": null- }+ ]}],"refresh": "30s",- "schemaVersion": 30,- "style": "dark",- "tags": [],+ "schemaVersion": 39,+ "tags": [+ "seaweedfs"+ ],"templating": {"list": [{- "current": {- "selected": true,- "text": "default",- "value": "default"- },+ "current": {},"hide": 0,"includeAll": false,- "label": "Datasource",+ "label": "Data source","multi": false,"name": "DS_PROMETHEUS","options": [],@@ -3504,40 +8603,42 @@"regex": "","skipUrlSync": false,"type": "datasource"+ },+ {+ "current": {},+ "datasource": {+ "type": "prometheus",+ "uid": "${DS_PROMETHEUS}"+ },+ "definition": "label_values(SeaweedFS_build_info, cluster)",+ "hide": 0,+ "includeAll": true,+ "allValue": ".*",+ "label": "Cluster",+ "multi": true,+ "name": "cluster",+ "options": [],+ "refresh": 2,+ "query": {+ "qryType": 1,+ "query": "label_values(SeaweedFS_build_info, cluster)",+ "refId": "PrometheusVariableQueryEditor-VariableQuery"+ },+ "regex": "",+ "skipUrlSync": false,+ "sort": 1,+ "type": "query"}]},"time": {- "from": "now-3h",+ "from": "now-6h","to": "now"},- "timepicker": {- "refresh_intervals": [- "5s",- "10s",- "30s",- "1m",- "5m",- "15m",- "30m",- "1h",- "2h",- "1d"- ],- "time_options": [- "5m",- "15m",- "1h",- "6h",- "12h",- "24h",- "2d",- "7d",- "30d"- ]- },- "timezone": "browser",+ "timepicker": {},+ "timezone": "","title": "SeaweedFS","uid": "nh02dOVnz",- "version": 2+ "version": 1,+ "weekStart": ""}