SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.
Go blob-storagecloud-drivedistributed-file-systemdistributed-storageerasure-codingfusehadoop-hdfshdfskuberneteslancelancedbobject-storageposixs3s3-storageseaweedfstiered-file-systemvector-search
telemetry: per-cluster size over time on the dashboard (#10417)
Chris Luc194924d13de
diff --git a/telemetry/README.md b/telemetry/README.mdindex 6e1db0aebd..f07f43b883 100644--- a/telemetry/README.md+++ b/telemetry/README.md@@ -177,6 +177,9 @@ GET /api/metrics?days=30# Get one cluster's daily usage history (disk bytes, volumes, volume servers)GET /api/history?cluster_id=<uuid>&days=90++# Get per-cluster disk usage over time, largest first, the rest summed as "other"+GET /api/cluster-sizes?days=30&limit=20```### Monitoringdiff --git a/telemetry/server/api/handlers.go b/telemetry/server/api/handlers.goindex b3faf234d0..a4c767ebc4 100644--- a/telemetry/server/api/handlers.go+++ b/telemetry/server/api/handlers.go@@ -150,6 +150,30 @@ func (h *Handler) GetMetrics(w http.ResponseWriter, r *http.Request) {json.NewEncoder(w).Encode(metrics)}+func (h *Handler) GetClusterSizes(w http.ResponseWriter, r *http.Request) {+ if r.Method != http.MethodGet {+ http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)+ return+ }++ days := 30 // default+ if daysStr := r.URL.Query().Get("days"); daysStr != "" {+ if d, err := strconv.Atoi(daysStr); err == nil && d > 0 && d <= 365 {+ days = d+ }+ }++ limit := 20 // default+ if limitStr := r.URL.Query().Get("limit"); limitStr != "" {+ if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 1000 {+ limit = l+ }+ }++ w.Header().Set("Content-Type", "application/json")+ json.NewEncoder(w).Encode(h.storage.GetClusterSizeSeries(days, limit))+}+func (h *Handler) GetHistory(w http.ResponseWriter, r *http.Request) {if r.Method != http.MethodGet {http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)diff --git a/telemetry/server/dashboard/dashboard.go b/telemetry/server/dashboard/dashboard.goindex 78301acc59..c977c4101d 100644--- a/telemetry/server/dashboard/dashboard.go+++ b/telemetry/server/dashboard/dashboard.go@@ -69,6 +69,10 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {font-weight: bold;margin-bottom: 15px;}+ .chart-subtitle {+ color: #666;+ margin: -10px 0 15px;+ }.loading {text-align: center;padding: 40px;@@ -159,6 +163,14 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {<canvas id="diskChart" width="400" height="200"></canvas></div>+ <div class="chart-container">+ <div class="chart-title">Cluster Sizes Over Time</div>+ <div class="chart-subtitle" id="clusterSizesTotal"></div>+ <div style="position: relative; height: 420px;">+ <canvas id="clusterSizeChart"></canvas>+ </div>+ </div>+<div class="chart-container"><div class="chart-title">Per-Cluster History</div><div class="cluster-lookup">@@ -177,6 +189,7 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {<script>let charts = {};+ let clusterSizeIds = [];async function loadDashboard() {try {@@ -188,8 +201,13 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {const metricsResponse = await fetch('/api/metrics?days=30');const metrics = await metricsResponse.json();+ // Load per-cluster sizes over time+ const sizesResponse = await fetch('/api/cluster-sizes?days=30&limit=20');+ const sizes = await sizesResponse.json();+updateStats(stats);updateCharts(stats, metrics);+ updateClusterSizes(sizes);document.getElementById('loading').style.display = 'none';document.getElementById('dashboard').style.display = 'block';@@ -292,6 +310,117 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {});}+ // Decimal units, so the round numbers the chart picks for its ticks+ // come out as round labels.+ function formatBytes(bytes) {+ const units = ['B', 'kB', 'MB', 'GB', 'TB', 'PB'];+ let value = bytes || 0, unit = 0;+ while (value >= 1000 && unit < units.length - 1) {+ value /= 1000;+ unit++;+ }+ return (unit === 0 ? value : value.toFixed(value >= 100 ? 0 : 1)) + ' ' + units[unit];+ }++ // One stacked band per cluster over time: the band is that cluster's+ // size, the top of the stack is the fleet total. Clusters beyond the+ // requested limit are summed into a trailing "other" band so the stack+ // still adds up.+ function updateClusterSizes(series) {+ const clusters = series.clusters || [];+ const dates = series.dates || [];+ const count = series.cluster_count || 0;++ document.getElementById('clusterSizesTotal').textContent =+ formatBytes(series.total_disk) + ' across ' + count + ' cluster' + (count === 1 ? '' : 's') ++ ' on ' + (dates[dates.length - 1] || 'no data');++ clusterSizeIds = clusters.map(c => c.cluster_id);+ const datasets = clusters.map((c, i) => {+ // Evenly spaced hues keep neighbouring bands distinguishable.+ const hue = Math.round(i * 360 / clusters.length);+ return band(c.cluster_id.slice(0, 8), c.disk,+ 'hsl(' + hue + ', 65%, 45%)', 'hsla(' + hue + ', 65%, 55%, 0.75)');+ });+ if (series.other) {+ clusterSizeIds.push(null);+ datasets.push(band('other (' + series.other.count + ')', series.other.disk,+ '#9E9E9E', 'rgba(158, 158, 158, 0.6)'));+ }++ const ctx = document.getElementById('clusterSizeChart').getContext('2d');+ if (charts.clusterSizeChart) {+ charts.clusterSizeChart.destroy();+ }+ charts.clusterSizeChart = new Chart(ctx, {+ type: 'line',+ data: { labels: dates, datasets: datasets },+ options: {+ responsive: true,+ maintainAspectRatio: false,+ interaction: { mode: 'band', intersect: false },+ onClick: (event, elements) => {+ const id = elements.length && clusterSizeIds[elements[0].datasetIndex];+ if (id) {+ document.getElementById('clusterIdInput').value = id;+ loadClusterHistory();+ }+ },+ plugins: {+ legend: { position: 'bottom', labels: { boxWidth: 12, font: { size: 11 } } },+ tooltip: {+ callbacks: {+ label: item => {+ const id = clusterSizeIds[item.datasetIndex];+ return (id || item.dataset.label) + ': ' + formatBytes(item.raw);+ }+ }+ }+ },+ scales: {+ x: { ticks: { maxRotation: 0, autoSkip: true, maxTicksLimit: 12 } },+ y: {+ stacked: true,+ beginAtZero: true,+ ticks: { callback: value => formatBytes(value) }+ }+ }+ }+ });+ }++ // Hover and click resolve to the band the pointer is inside. Chart.js's+ // built-in modes match the nearest line, which on a stack of thin bands+ // is rarely the one under the pointer.+ Chart.Interaction.modes.band = function(chart, event) {+ const last = chart.data.labels.length - 1;+ if (last < 0) return [];+ const index = Math.min(Math.max(Math.round(chart.scales.x.getValueForPixel(event.x)), 0), last);+ const value = chart.scales.y.getValueForPixel(event.y);+ let stacked = 0;+ for (let d = 0; d < chart.data.datasets.length; d++) {+ stacked += chart.data.datasets[d].data[index] || 0;+ if (value <= stacked) {+ return [{ element: chart.getDatasetMeta(d).data[index], datasetIndex: d, index: index }];+ }+ }+ return [];+ };++ function band(label, data, borderColor, backgroundColor) {+ return {+ label: label,+ data: data,+ borderColor: borderColor,+ backgroundColor: backgroundColor,+ borderWidth: 1,+ pointRadius: 0,+ pointHitRadius: 8,+ fill: true,+ tension: 0.1+ };+ }+async function loadClusterHistory() {const id = document.getElementById('clusterIdInput').value.trim();const errorDiv = document.getElementById('clusterHistoryError');diff --git a/telemetry/server/main.go b/telemetry/server/main.goindex 05ebec8dc3..0383c1d280 100644--- a/telemetry/server/main.go+++ b/telemetry/server/main.go@@ -76,6 +76,7 @@ func main() {mux.HandleFunc("/api/instances", corsMiddleware(logMiddleware(apiHandler.GetInstances)))mux.HandleFunc("/api/metrics", corsMiddleware(logMiddleware(apiHandler.GetMetrics)))mux.HandleFunc("/api/history", corsMiddleware(logMiddleware(apiHandler.GetHistory)))+ mux.HandleFunc("/api/cluster-sizes", corsMiddleware(logMiddleware(apiHandler.GetClusterSizes)))// Dashboard (optional)if *enableDashboard {diff --git a/telemetry/server/storage/history.go b/telemetry/server/storage/history.goindex eff68bc03f..2ec6a1a930 100644--- a/telemetry/server/storage/history.go+++ b/telemetry/server/storage/history.go@@ -10,6 +10,9 @@ import (// on before it counts as confirmed in the aggregated stats.const confirmDays = 2+// activeDays is how recently a cluster must have reported to count as active.+const activeDays = 7+// HistorySample is one retained data point of a cluster's daily reports.// Tags are kept short because thousands of samples end up in the state file.type HistorySample struct {diff --git a/telemetry/server/storage/prometheus.go b/telemetry/server/storage/prometheus.goindex 15e040f9dd..10092e6488 100644--- a/telemetry/server/storage/prometheus.go+++ b/telemetry/server/storage/prometheus.go@@ -207,7 +207,7 @@ func (s *PrometheusStorage) GetMetrics(days int) (map[string]interface{}, error)func (s *PrometheusStorage) updateStats() {now := time.Now()- last7Days := now.AddDate(0, 0, -7)+ last7Days := now.AddDate(0, 0, -activeDays)last30Days := now.AddDate(0, 0, -30)totalInstances := 0diff --git a/telemetry/server/storage/sizes.go b/telemetry/server/storage/sizes.gonew file mode 100644index 0000000000..aa2f7f287a--- /dev/null+++ b/telemetry/server/storage/sizes.go@@ -0,0 +1,106 @@+package storage++import (+ "sort"+ "time"+)++// ClusterSeries is one cluster's daily disk usage, aligned to the shared date+// axis of the enclosing ClusterSizeSeries.+type ClusterSeries struct {+ ClusterId string `json:"cluster_id"`+ Disk []uint64 `json:"disk"`+}++// OtherSeries is the clusters beyond the caller's limit, summed per day so a+// stacked chart still adds up to the fleet total.+type OtherSeries struct {+ Count int `json:"count"`+ Disk []uint64 `json:"disk"`+}++// ClusterSizeSeries is per-cluster disk usage over time: one value per cluster+// per day, largest cluster first, ranked by their most recent day.+type ClusterSizeSeries struct {+ Dates []string `json:"dates"`+ Clusters []ClusterSeries `json:"clusters"`+ Other *OtherSeries `json:"other,omitempty"`+ ClusterCount int `json:"cluster_count"`+ TotalDisk uint64 `json:"total_disk"` // across all clusters on the last day+}++// GetClusterSizeSeries returns the last `days` days of per-cluster disk usage.+// Clusters beyond `limit` are folded into Other. Clusters report roughly once+// a day at no fixed hour, so a day without a report carries the previous value+// forward rather than dropping to zero; a cluster that stopped reporting+// altogether ends at its last sample instead of holding capacity forever.+func (s *PrometheusStorage) GetClusterSizeSeries(days, limit int) ClusterSizeSeries {+ s.mu.RLock()+ defer s.mu.RUnlock()++ now := time.Now().UTC()+ today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)+ dayOf := make(map[string]int, days)+ series := ClusterSizeSeries{Dates: make([]string, days)}+ for i := range series.Dates {+ series.Dates[i] = today.AddDate(0, 0, i-days+1).Format("2006-01-02")+ dayOf[series.Dates[i]] = i+ }+ activeSince := now.AddDate(0, 0, -activeDays).Unix()++ for id, history := range s.histories {+ disk := make([]uint64, days)+ reported := make([]bool, days)+ first, last := -1, -1+ for _, sample := range history {+ i, ok := dayOf[time.Unix(sample.Ts, 0).UTC().Format("2006-01-02")]+ if !ok {+ continue+ }+ disk[i], reported[i] = sample.TotalDiskBytes, true+ if first < 0 {+ first = i+ }+ last = i+ if sample.Ts >= activeSince {+ last = days - 1 // still reporting, so hold its size to the right edge+ }+ }+ if first < 0 {+ continue+ }+ for i := first + 1; i <= last; i++ {+ if !reported[i] {+ disk[i] = disk[i-1]+ }+ }+ series.Clusters = append(series.Clusters, ClusterSeries{ClusterId: id, Disk: disk})+ series.TotalDisk += disk[days-1]+ }+ series.ClusterCount = len(series.Clusters)++ // Rank by the latest day so the stack reads largest-first at its right+ // edge, tie-breaking on id to keep the order stable across refreshes.+ sort.Slice(series.Clusters, func(i, j int) bool {+ a, b := series.Clusters[i], series.Clusters[j]+ if a.Disk[days-1] != b.Disk[days-1] {+ return a.Disk[days-1] > b.Disk[days-1]+ }+ return a.ClusterId < b.ClusterId+ })++ if limit > 0 && len(series.Clusters) > limit {+ other := OtherSeries{+ Count: len(series.Clusters) - limit,+ Disk: make([]uint64, days),+ }+ for _, c := range series.Clusters[limit:] {+ for i, v := range c.Disk {+ other.Disk[i] += v+ }+ }+ series.Clusters = series.Clusters[:limit]+ series.Other = &other+ }+ return series+}diff --git a/telemetry/server/storage/sizes_test.go b/telemetry/server/storage/sizes_test.gonew file mode 100644index 0000000000..54af4fd438--- /dev/null+++ b/telemetry/server/storage/sizes_test.go@@ -0,0 +1,122 @@+package storage++import (+ "testing"+ "time"++ "github.com/prometheus/client_golang/prometheus"+ "github.com/seaweedfs/seaweedfs/telemetry/proto"+)++// seedHistory gives a cluster one sample per listed day offset (0 is today).+func seedHistory(s *PrometheusStorage, id string, disk uint64, dayOffsets ...int) {+ s.mu.Lock()+ defer s.mu.Unlock()+ for _, offset := range dayOffsets {+ s.histories[id] = append(s.histories[id], HistorySample{+ Ts: time.Now().AddDate(0, 0, offset).Unix(),+ TotalDiskBytes: disk,+ })+ }+}++func TestClusterSizeSeries(t *testing.T) {+ s := newPrometheusStorage(prometheus.NewRegistry())++ // Reported every day of the window.+ seedHistory(s, "daily", 300, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0)+ // Reported two days ago and not since: still active, so its size is held+ // to the right edge instead of dropping out of the stack.+ seedHistory(s, "lagging", 200, -2)+ // Stopped reporting past the active window: its own days only.+ seedHistory(s, "gone", 900, -9, -8)++ series := s.GetClusterSizeSeries(10, 0)+ if len(series.Dates) != 10 {+ t.Fatalf("dates = %v, want 10 days", series.Dates)+ }+ if series.ClusterCount != 3 {+ t.Fatalf("cluster_count = %d, want 3", series.ClusterCount)+ }++ byId := map[string][]uint64{}+ for _, c := range series.Clusters {+ byId[c.ClusterId] = c.Disk+ }+ if got := byId["daily"]; !equal(got, []uint64{300, 300, 300, 300, 300, 300, 300, 300, 300, 300}) {+ t.Errorf("daily = %v, want 300 every day", got)+ }+ if got := byId["lagging"]; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 0, 200, 200, 200}) {+ t.Errorf("lagging = %v, want its size carried to the right edge", got)+ }+ if got := byId["gone"]; !equal(got, []uint64{900, 900, 0, 0, 0, 0, 0, 0, 0, 0}) {+ t.Errorf("gone = %v, want no capacity after its last report", got)+ }++ // The total is the last day's stack height: daily + lagging, not gone.+ if series.TotalDisk != 500 {+ t.Errorf("total_disk = %d, want 500", series.TotalDisk)+ }+ // Largest on the last day comes first so the stack reads top-down.+ if series.Clusters[0].ClusterId != "daily" {+ t.Errorf("order = %s first, want daily", series.Clusters[0].ClusterId)+ }++ // Clusters past the limit are summed into "other", per day.+ series = s.GetClusterSizeSeries(10, 1)+ if len(series.Clusters) != 1 || series.Clusters[0].ClusterId != "daily" {+ t.Fatalf("limited clusters = %+v, want just daily", series.Clusters)+ }+ if series.Other == nil || series.Other.Count != 2 {+ t.Fatalf("other = %+v, want 2 clusters", series.Other)+ }+ if !equal(series.Other.Disk, []uint64{900, 900, 0, 0, 0, 0, 0, 200, 200, 200}) {+ t.Errorf("other = %v, want lagging+gone summed per day", series.Other.Disk)+ }+ if series.ClusterCount != 3 || series.TotalDisk != 500 {+ t.Errorf("limit changed totals: count=%d disk=%d, want 3/500", series.ClusterCount, series.TotalDisk)+ }+}++// A report that lands after an earlier one on the same day replaces it, so the+// series shows one value per cluster per day.+func TestClusterSizeSeriesUsesLatestDailySample(t *testing.T) {+ s := newPrometheusStorage(prometheus.NewRegistry())++ data := &proto.TelemetryData{+ TopologyId: "aaaaaaaa-0000-0000-0000-000000000001",+ Version: "4.40",+ Os: "linux/amd64",+ TotalDiskBytes: 100,+ }+ if err := s.StoreTelemetry(data); err != nil {+ t.Fatal(err)+ }+ data.TotalDiskBytes = 700+ if err := s.StoreTelemetry(data); err != nil {+ t.Fatal(err)+ }++ series := s.GetClusterSizeSeries(3, 0)+ if len(series.Clusters) != 1 {+ t.Fatalf("clusters = %+v, want 1", series.Clusters)+ }+ if got := series.Clusters[0].Disk; !equal(got, []uint64{0, 0, 700}) {+ t.Errorf("disk = %v, want today's latest sample only", got)+ }+ if series.TotalDisk != 700 {+ t.Errorf("total_disk = %d, want 700", series.TotalDisk)+ }+}++func equal(a, b []uint64) bool {+ if len(a) != len(b) {+ return false+ }+ for i := range a {+ if a[i] != b[i] {+ return false+ }+ }+ return true+}