Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Read PMG queue counts from qshape, not the queue index

rcourtman5842fd060055
diff --git a/internal/monitoring/monitor_pmg_test.go b/internal/monitoring/monitor_pmg_test.go
index 46a9944207..14a7a7070e 100644
--- a/internal/monitoring/monitor_pmg_test.go
+++ b/internal/monitoring/monitor_pmg_test.go
@@ -64,9 +64,18 @@ func TestPollPMGInstancePopulatesState(t *testing.T) {
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"count":2,"avgbytes":0,"mbytes":0}}`)
- case "/api2/json/nodes/mail-gateway/postfix/queue":
+ case "/api2/json/nodes/mail-gateway/postfix/qshape":
+ totals := map[string]string{"active": "5", "deferred": "2", "hold": "0", "incoming": "0"}
+ total, ok := totals[r.URL.Query().Get("queue")]
+ if !ok {
+ t.Fatalf("unexpected qshape queue: %s", r.URL.RawQuery)
+ }
+ oldest := "0"
+ if r.URL.Query().Get("queue") == "deferred" {
+ oldest = total
+ }
w.Header().Set("Content-Type", "application/json")
- fmt.Fprint(w, `{"data":{"active":5,"deferred":2,"hold":0,"maildrop":0}}`)
+ fmt.Fprintf(w, `{"data":[{"domain":"TOTAL","total":"%s","5m":"0","10m":"0","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"%s","1280m+":"0"}]}`, total, oldest)
case "/api2/json/nodes/mail-gateway/backup":
w.Header().Set("Content-Type", "application/json")
@@ -154,6 +163,13 @@ func TestPollPMGInstancePopulatesState(t *testing.T) {
t.Fatalf("expected quarantine counts, got %+v", instance.Quarantine)
}
+ if len(instance.Nodes) != 1 || instance.Nodes[0].QueueStatus == nil {
+ t.Fatalf("expected node queue status, got %+v", instance.Nodes)
+ }
+ if queue := instance.Nodes[0].QueueStatus; queue.Active != 5 || queue.Deferred != 2 || queue.Total != 7 || queue.OldestAge != 640*60 {
+ t.Fatalf("unexpected queue status: %+v", queue)
+ }
+
if health := snapshot.ConnectionHealth["pmg-primary"]; !health {
t.Fatalf("expected connection health tracked as healthy, got %v", health)
}
diff --git a/pkg/pmg/client.go b/pkg/pmg/client.go
index d2d5f4d7b8..7d021d7168 100644
--- a/pkg/pmg/client.go
+++ b/pkg/pmg/client.go
@@ -143,12 +143,50 @@ type QuarantineStatus struct {
Megabytes flexibleFloat `json:"mbytes"`
}
+// QueueStatusEntry summarises one node's Postfix queues. PMG has no single
+// queue-summary endpoint, so GetQueueStatus assembles it from qshape.
type QueueStatusEntry struct {
Active flexibleInt `json:"active"`
Deferred flexibleInt `json:"deferred"`
Hold flexibleInt `json:"hold"`
Incoming flexibleInt `json:"incoming"`
- OldestAge flexibleInt `json:"oldest_age,omitempty"` // Age of oldest message in seconds
+ OldestAge flexibleInt `json:"oldest_age,omitempty"` // Lower bound, in seconds, on the oldest active, deferred or incoming message
+}
+
+// queueShapeRow is one row of PMG's qshape output. Counts are recipients, and
+// each bucket holds entries older than the previous bucket's limit and younger
+// than its own, in minutes; 1280m+ has no upper limit.
+type queueShapeRow struct {
+ Domain string `json:"domain"`
+ Total flexibleInt `json:"total"`
+ B5 flexibleInt `json:"5m"`
+ B10 flexibleInt `json:"10m"`
+ B20 flexibleInt `json:"20m"`
+ B40 flexibleInt `json:"40m"`
+ B80 flexibleInt `json:"80m"`
+ B160 flexibleInt `json:"160m"`
+ B320 flexibleInt `json:"320m"`
+ B640 flexibleInt `json:"640m"`
+ B1280 flexibleInt `json:"1280m"`
+ B1280Up flexibleInt `json:"1280m+"`
+}
+
+// minAgeSeconds returns the lower limit of the oldest non-empty bucket, which
+// is the most qshape proves about the oldest entry's age.
+func (r queueShapeRow) minAgeSeconds() int64 {
+ buckets := []struct {
+ count flexibleInt
+ minMinutes int64
+ }{
+ {r.B1280Up, 1280}, {r.B1280, 640}, {r.B640, 320}, {r.B320, 160}, {r.B160, 80},
+ {r.B80, 40}, {r.B40, 20}, {r.B20, 10}, {r.B10, 5},
+ }
+ for _, b := range buckets {
+ if b.count.Int64() > 0 {
+ return b.minMinutes * 60
+ }
+ }
+ return 0
}
// DomainStatisticsEntry represents aggregated statistics for a single domain.
@@ -554,16 +592,46 @@ func (c *Client) GetQuarantineStatus(ctx context.Context, category string) (*Qua
return resp.Data, nil
}
+// GetQueueStatus reads each Postfix queue's TOTAL row from qshape. The
+// /postfix/queue path is only a directory index of queue names (#2960).
func (c *Client) GetQueueStatus(ctx context.Context, node string) (*QueueStatusEntry, error) {
- path := fmt.Sprintf("/nodes/%s/postfix/queue", node)
- var resp apiResponse[*QueueStatusEntry]
- if err := c.getJSON(ctx, path, nil, &resp); err != nil {
- return nil, err
+ status := &QueueStatusEntry{}
+ queues := []struct {
+ name string
+ count *flexibleInt
+ ageless bool
+ }{
+ {name: "active", count: &status.Active},
+ {name: "deferred", count: &status.Deferred},
+ // Held mail waits for an operator by design, so its age is not a delivery delay.
+ {name: "hold", count: &status.Hold, ageless: true},
+ {name: "incoming", count: &status.Incoming},
+ }
+ path := fmt.Sprintf("/nodes/%s/postfix/qshape", url.PathEscape(node))
+ for _, queue := range queues {
+ var resp apiResponse[[]queueShapeRow]
+ if err := c.getJSON(ctx, path, url.Values{"queue": {queue.name}}, &resp); err != nil {
+ return nil, fmt.Errorf("%s queue: %w", queue.name, err)
+ }
+ total, ok := queueShapeTotal(resp.Data)
+ if !ok {
+ return nil, fmt.Errorf("PMG GetQueueStatus %s queue response missing TOTAL row", queue.name)
+ }
+ *queue.count = total.Total
+ if age := total.minAgeSeconds(); !queue.ageless && age > status.OldestAge.Int64() {
+ status.OldestAge = flexibleInt(age)
+ }
}
- if resp.Data == nil {
- return nil, fmt.Errorf("PMG GetQueueStatus response missing data")
+ return status, nil
+}
+
+func queueShapeTotal(rows []queueShapeRow) (queueShapeRow, bool) {
+ for _, row := range rows {
+ if row.Domain == "TOTAL" {
+ return row, true
+ }
}
- return resp.Data, nil
+ return queueShapeRow{}, false
}
// ListBackups returns configuration backup archives available on a PMG node.
diff --git a/pkg/pmg/client_test.go b/pkg/pmg/client_test.go
index 778ff30c3d..d0facd4661 100644
--- a/pkg/pmg/client_test.go
+++ b/pkg/pmg/client_test.go
@@ -401,9 +401,20 @@ func TestMailEndpointsHandleNullAndStringValues(t *testing.T) {
case "/api2/json/quarantine/virusstatus":
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `{"data":{"count":"4","avgbytes":"256","mbytes":"0.25"}}`)
- case "/api2/json/nodes/mail/postfix/queue":
+ case "/api2/json/nodes/mail/postfix/qshape":
+ // PMG emits qshape counts as strings, with the TOTAL row first.
+ rows := map[string]string{
+ "active": `{"domain":"TOTAL","total":"3","5m":"1","10m":"2","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"0","1280m+":"0"},{"domain":"example.com","total":"3","5m":"1","10m":"2","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"0","1280m+":"0"}`,
+ "deferred": `{"domain":"TOTAL","total":"0","5m":"0","10m":"0","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"0","1280m+":"0"}`,
+ "hold": `{"domain":"TOTAL","total":"1","5m":"0","10m":"0","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"0","1280m+":"1"}`,
+ "incoming": `{"domain":"TOTAL","total":"2","5m":"2","10m":"0","20m":"0","40m":"0","80m":"0","160m":"0","320m":"0","640m":"0","1280m":"0","1280m+":"0"}`,
+ }
+ row, ok := rows[r.URL.Query().Get("queue")]
+ if !ok {
+ t.Fatalf("unexpected qshape queue: %s", r.URL.RawQuery)
+ }
w.Header().Set("Content-Type", "application/json")
- fmt.Fprint(w, `{"data":{"active":"3","deferred":null,"hold":"1","incoming":"2","oldest_age":"600"}}`)
+ fmt.Fprintf(w, `{"data":[%s]}`, row)
default:
t.Fatalf("unexpected path: %s", r.URL.Path)
}
@@ -475,8 +486,10 @@ func TestMailEndpointsHandleNullAndStringValues(t *testing.T) {
if queue.Active.Int() != 3 || queue.Deferred.Int() != 0 || queue.Hold.Int() != 1 || queue.Incoming.Int() != 2 {
t.Fatalf("unexpected queue values: %+v", queue)
}
- if queue.OldestAge.Int64() != 600 {
- t.Fatalf("expected oldest age 600, got %d", queue.OldestAge.Int64())
+ // Active mail in the 5-10 minute bucket is the oldest delivery delay; the
+ // held message is older but waits for an operator.
+ if queue.OldestAge.Int64() != 300 {
+ t.Fatalf("expected oldest age 300, got %d", queue.OldestAge.Int64())
}
}
@@ -836,7 +849,9 @@ type pmgCollectionTransport func(*http.Request) (*http.Response, error)
func (f pmgCollectionTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestPMGCollectionMissingObjectData(t *testing.T) {
- for _, body := range []string{`{}`, `{"data":null}`, `{"data":0}`, `{"data":[]}`, `invalid`} {
+ // The last body is PMG's /postfix/queue directory index, which Pulse once
+ // decoded as queue counts (#2960).
+ for _, body := range []string{`{}`, `{"data":null}`, `{"data":0}`, `{"data":[]}`, `invalid`, `{"data":[{"name":"deferred"},{"name":"active"},{"name":"incoming"},{"name":"hold"}]}`} {
t.Run(body, func(t *testing.T) {
client := &Client{baseURL: "https://example.invalid", auth: auth{tokenName: "synthetic", tokenValue: "synthetic", user: "fixture", realm: "pmg"}, httpClient: &http.Client{Transport: pmgCollectionTransport(func(r *http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
@@ -858,7 +873,11 @@ func TestPMGCollectionMissingObjectData(t *testing.T) {
}
func TestPMGCollectionObservedZeroData(t *testing.T) {
client := &Client{baseURL: "https://example.invalid", auth: auth{tokenName: "synthetic", tokenValue: "synthetic", user: "fixture", realm: "pmg"}, httpClient: &http.Client{Transport: pmgCollectionTransport(func(r *http.Request) (*http.Response, error) {
- return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"data":{"count":0,"active":0,"deferred":0,"hold":0,"incoming":0}}`)), Request: r}, nil
+ body := `{"data":{"count":0}}`
+ if strings.HasSuffix(r.URL.Path, "/postfix/qshape") {
+ body = `{"data":[{"domain":"TOTAL","total":0,"5m":0,"10m":0,"20m":0,"40m":0,"80m":0,"160m":0,"320m":0,"640m":0,"1280m":0,"1280m+":0}]}`
+ }
+ return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
})}}
if v, err := client.GetMailStatistics(context.Background(), ""); err != nil || v == nil || v.Count.Float64() != 0 {
t.Fatalf("mail=%+v err=%v", v, err)