Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Read PMG queue counts from qshape, not the queue index
rcourtman5842fd060055
diff --git a/internal/monitoring/monitor_pmg_test.go b/internal/monitoring/monitor_pmg_test.goindex 46a9944207..14a7a7070e 100644--- a/internal/monitoring/monitor_pmg_test.go+++ b/internal/monitoring/monitor_pmg_test.go@@ -64,9 +64,18 @@ func TestPollPMGInstancePopulatesState(t *testing.T) {w.Header().Set("Content-Type", "application/json")fmt.Fprint(w, `{"data":{"count":2,"avgbytes":0,"mbytes":0}}`)- case "/api2/json/nodes/mail-gateway/postfix/queue":+ case "/api2/json/nodes/mail-gateway/postfix/qshape":+ totals := map[string]string{"active": "5", "deferred": "2", "hold": "0", "incoming": "0"}+ total, ok := totals[r.URL.Query().Get("queue")]+ if !ok {+ t.Fatalf("unexpected qshape queue: %s", r.URL.RawQuery)+ }+ oldest := "0"+ if r.URL.Query().Get("queue") == "deferred" {+ oldest = total+ }w.Header().Set("Content-Type", "application/json")- fmt.Fprint(w, `{"data":{"active":5,"deferred":2,"hold":0,"maildrop":0}}`)+ fmt.Fprintf(w, `{"data":[{"domain":"TOTAL","total":"%s","5m":"0","10m":"0","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"%s","1280m+":"0"}]}`, total, oldest)case "/api2/json/nodes/mail-gateway/backup":w.Header().Set("Content-Type", "application/json")@@ -154,6 +163,13 @@ func TestPollPMGInstancePopulatesState(t *testing.T) {t.Fatalf("expected quarantine counts, got %+v", instance.Quarantine)}+ if len(instance.Nodes) != 1 || instance.Nodes[0].QueueStatus == nil {+ t.Fatalf("expected node queue status, got %+v", instance.Nodes)+ }+ if queue := instance.Nodes[0].QueueStatus; queue.Active != 5 || queue.Deferred != 2 || queue.Total != 7 || queue.OldestAge != 640*60 {+ t.Fatalf("unexpected queue status: %+v", queue)+ }+if health := snapshot.ConnectionHealth["pmg-primary"]; !health {t.Fatalf("expected connection health tracked as healthy, got %v", health)}diff --git a/pkg/pmg/client.go b/pkg/pmg/client.goindex d2d5f4d7b8..7d021d7168 100644--- a/pkg/pmg/client.go+++ b/pkg/pmg/client.go@@ -143,12 +143,50 @@ type QuarantineStatus struct {Megabytes flexibleFloat `json:"mbytes"`}+// QueueStatusEntry summarises one node's Postfix queues. PMG has no single+// queue-summary endpoint, so GetQueueStatus assembles it from qshape.type QueueStatusEntry struct {Active flexibleInt `json:"active"`Deferred flexibleInt `json:"deferred"`Hold flexibleInt `json:"hold"`Incoming flexibleInt `json:"incoming"`- OldestAge flexibleInt `json:"oldest_age,omitempty"` // Age of oldest message in seconds+ OldestAge flexibleInt `json:"oldest_age,omitempty"` // Lower bound, in seconds, on the oldest active, deferred or incoming message+}++// queueShapeRow is one row of PMG's qshape output. Counts are recipients, and+// each bucket holds entries older than the previous bucket's limit and younger+// than its own, in minutes; 1280m+ has no upper limit.+type queueShapeRow struct {+ Domain string `json:"domain"`+ Total flexibleInt `json:"total"`+ B5 flexibleInt `json:"5m"`+ B10 flexibleInt `json:"10m"`+ B20 flexibleInt `json:"20m"`+ B40 flexibleInt `json:"40m"`+ B80 flexibleInt `json:"80m"`+ B160 flexibleInt `json:"160m"`+ B320 flexibleInt `json:"320m"`+ B640 flexibleInt `json:"640m"`+ B1280 flexibleInt `json:"1280m"`+ B1280Up flexibleInt `json:"1280m+"`+}++// minAgeSeconds returns the lower limit of the oldest non-empty bucket, which+// is the most qshape proves about the oldest entry's age.+func (r queueShapeRow) minAgeSeconds() int64 {+ buckets := []struct {+ count flexibleInt+ minMinutes int64+ }{+ {r.B1280Up, 1280}, {r.B1280, 640}, {r.B640, 320}, {r.B320, 160}, {r.B160, 80},+ {r.B80, 40}, {r.B40, 20}, {r.B20, 10}, {r.B10, 5},+ }+ for _, b := range buckets {+ if b.count.Int64() > 0 {+ return b.minMinutes * 60+ }+ }+ return 0}// DomainStatisticsEntry represents aggregated statistics for a single domain.@@ -554,16 +592,46 @@ func (c *Client) GetQuarantineStatus(ctx context.Context, category string) (*Quareturn resp.Data, nil}+// GetQueueStatus reads each Postfix queue's TOTAL row from qshape. The+// /postfix/queue path is only a directory index of queue names (#2960).func (c *Client) GetQueueStatus(ctx context.Context, node string) (*QueueStatusEntry, error) {- path := fmt.Sprintf("/nodes/%s/postfix/queue", node)- var resp apiResponse[*QueueStatusEntry]- if err := c.getJSON(ctx, path, nil, &resp); err != nil {- return nil, err+ status := &QueueStatusEntry{}+ queues := []struct {+ name string+ count *flexibleInt+ ageless bool+ }{+ {name: "active", count: &status.Active},+ {name: "deferred", count: &status.Deferred},+ // Held mail waits for an operator by design, so its age is not a delivery delay.+ {name: "hold", count: &status.Hold, ageless: true},+ {name: "incoming", count: &status.Incoming},+ }+ path := fmt.Sprintf("/nodes/%s/postfix/qshape", url.PathEscape(node))+ for _, queue := range queues {+ var resp apiResponse[[]queueShapeRow]+ if err := c.getJSON(ctx, path, url.Values{"queue": {queue.name}}, &resp); err != nil {+ return nil, fmt.Errorf("%s queue: %w", queue.name, err)+ }+ total, ok := queueShapeTotal(resp.Data)+ if !ok {+ return nil, fmt.Errorf("PMG GetQueueStatus %s queue response missing TOTAL row", queue.name)+ }+ *queue.count = total.Total+ if age := total.minAgeSeconds(); !queue.ageless && age > status.OldestAge.Int64() {+ status.OldestAge = flexibleInt(age)+ }}- if resp.Data == nil {- return nil, fmt.Errorf("PMG GetQueueStatus response missing data")+ return status, nil+}++func queueShapeTotal(rows []queueShapeRow) (queueShapeRow, bool) {+ for _, row := range rows {+ if row.Domain == "TOTAL" {+ return row, true+ }}- return resp.Data, nil+ return queueShapeRow{}, false}// ListBackups returns configuration backup archives available on a PMG node.diff --git a/pkg/pmg/client_test.go b/pkg/pmg/client_test.goindex 778ff30c3d..d0facd4661 100644--- a/pkg/pmg/client_test.go+++ b/pkg/pmg/client_test.go@@ -401,9 +401,20 @@ func TestMailEndpointsHandleNullAndStringValues(t *testing.T) {case "/api2/json/quarantine/virusstatus":w.Header().Set("Content-Type", "application/json")fmt.Fprint(w, `{"data":{"count":"4","avgbytes":"256","mbytes":"0.25"}}`)- case "/api2/json/nodes/mail/postfix/queue":+ case "/api2/json/nodes/mail/postfix/qshape":+ // PMG emits qshape counts as strings, with the TOTAL row first.+ rows := map[string]string{+ "active": `{"domain":"TOTAL","total":"3","5m":"1","10m":"2","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"0","1280m+":"0"},{"domain":"example.com","total":"3","5m":"1","10m":"2","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"0","1280m+":"0"}`,+ "deferred": `{"domain":"TOTAL","total":"0","5m":"0","10m":"0","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"0","1280m+":"0"}`,+ "hold": `{"domain":"TOTAL","total":"1","5m":"0","10m":"0","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"0","1280m+":"1"}`,+ "incoming": `{"domain":"TOTAL","total":"2","5m":"2","10m":"0","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"0","1280m+":"0"}`,+ }+ row, ok := rows[r.URL.Query().Get("queue")]+ if !ok {+ t.Fatalf("unexpected qshape queue: %s", r.URL.RawQuery)+ }w.Header().Set("Content-Type", "application/json")- fmt.Fprint(w, `{"data":{"active":"3","deferred":null,"hold":"1","incoming":"2","oldest_age":"600"}}`)+ fmt.Fprintf(w, `{"data":[%s]}`, row)default:t.Fatalf("unexpected path: %s", r.URL.Path)}@@ -475,8 +486,10 @@ func TestMailEndpointsHandleNullAndStringValues(t *testing.T) {if queue.Active.Int() != 3 || queue.Deferred.Int() != 0 || queue.Hold.Int() != 1 || queue.Incoming.Int() != 2 {t.Fatalf("unexpected queue values: %+v", queue)}- if queue.OldestAge.Int64() != 600 {- t.Fatalf("expected oldest age 600, got %d", queue.OldestAge.Int64())+ // Active mail in the 5-10 minute bucket is the oldest delivery delay; the+ // held message is older but waits for an operator.+ if queue.OldestAge.Int64() != 300 {+ t.Fatalf("expected oldest age 300, got %d", queue.OldestAge.Int64())}}@@ -836,7 +849,9 @@ type pmgCollectionTransport func(*http.Request) (*http.Response, error)func (f pmgCollectionTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }func TestPMGCollectionMissingObjectData(t *testing.T) {- for _, body := range []string{`{}`, `{"data":null}`, `{"data":0}`, `{"data":[]}`, `invalid`} {+ // The last body is PMG's /postfix/queue directory index, which Pulse once+ // decoded as queue counts (#2960).+ for _, body := range []string{`{}`, `{"data":null}`, `{"data":0}`, `{"data":[]}`, `invalid`, `{"data":[{"name":"deferred"},{"name":"active"},{"name":"incoming"},{"name":"hold"}]}`} {t.Run(body, func(t *testing.T) {client := &Client{baseURL: "https://example.invalid", auth: auth{tokenName: "synthetic", tokenValue: "synthetic", user: "fixture", realm: "pmg"}, httpClient: &http.Client{Transport: pmgCollectionTransport(func(r *http.Request) (*http.Response, error) {return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil@@ -858,7 +873,11 @@ func TestPMGCollectionMissingObjectData(t *testing.T) {}func TestPMGCollectionObservedZeroData(t *testing.T) {client := &Client{baseURL: "https://example.invalid", auth: auth{tokenName: "synthetic", tokenValue: "synthetic", user: "fixture", realm: "pmg"}, httpClient: &http.Client{Transport: pmgCollectionTransport(func(r *http.Request) (*http.Response, error) {- return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"data":{"count":0,"active":0,"deferred":0,"hold":0,"incoming":0}}`)), Request: r}, nil+ body := `{"data":{"count":0}}`+ if strings.HasSuffix(r.URL.Path, "/postfix/qshape") {+ body = `{"data":[{"domain":"TOTAL","total":0,"5m":0,"10m":0,"20m":0,"40m":0,"80m":0,"160m":0,"320m":0,"640m":0,"1280m":0,"1280m+":0}]}`+ }+ return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil})}}if v, err := client.GetMailStatistics(context.Background(), ""); err != nil || v == nil || v.Count.Float64() != 0 {t.Fatalf("mail=%+v err=%v", v, err)